Skip to content
Snippets Groups Projects
Commit f9bf5da1 authored by Recolic K's avatar Recolic K
Browse files

play toys

parent 919d7e9e
No related branches found
No related tags found
No related merge requests found
*.log
*.gz
*.pyobj
......@@ -3,7 +3,23 @@ import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
# [(Snapshot_Time, [(buy_price1, amount1), ...] , [(sell_price1, amount1), ...]), ...]
# [(1620457034392, [(56000, 0.01), (55900, 1), (55700, 30), ...] , [(57000, 0.01), (57100, 1), ...] ), (1620457034394, [...]), ...]
# The snapshots should has almost identical time-interval. Good for LSTM.
# Time axis: [history, older, newer, ..., latest]
realtime_shortterm_dataset_aggtrade = []
realtime_shortterm_dataset_aggtrade_size = 1024
# [(Trade_Time, PRICE, AMOUNT), ...]
# [(1620457034392, 56000, 0.5), (1620457034394, 56001, 0.05), ...]
# The trades usually have various time-interval. TODO: transform it to [(WeightedAvgPrice, Volume), ...] for every 1 minutes?
# Time axis: [history, older, newer, ..., latest]
realtime_shortterm_dataset_depth = []
realtime_shortterm_dataset_depth_size = 1024*1024
# The trading thread would not start working, before finish analysing longterm dataset.
# Time-interval for longterm dataset is 1 minute.
longterm_dataset = []
......
sample.py 0 → 100644
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from math import sin
#def prepare_sequence(seq, to_ix):
# idxs = [to_ix[w] for w in seq]
# return torch.tensor(idxs, dtype=torch.long)
def geni(i):
if i % 2 == 0:
return sin(i/3) + 0.05#, sin(i) - 0.08]
else:
return sin(i/3) - 0.1#, sin(i) - 0.1, sin(i)]
input_seq = [geni(i) for i in range(10240)]
EMBEDDING_DIM = 7
HIDDEN_DIM = 7
class LSTMTagger(nn.Module):
def __init__(self, input_dim, hidden_dim):
super(LSTMTagger, self).__init__()
self.hidden_dim = hidden_dim
# The LSTM takes word embeddings as inputs, and outputs hidden states
# with dimensionality hidden_dim.
self.lstm = nn.LSTM(1, hidden_dim)
# The linear layer that maps from hidden state space to tag space
self.out = nn.Linear(hidden_dim, 1)
def forward(self, sample_seq, lstm_state):
lstm_out, lstm_state = self.lstm(sample_seq.view(len(sample_seq), 1, 1))
score = self.out(torch.tanh(lstm_out[-1:]))
return score, lstm_state
model = LSTMTagger(128, 128)
loss_function = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=0.1)
# See what the scores are before training
# Note that element i,j of the output is the score for tag j for word i.
# Here we don't need to train, so the code is wrapped in torch.no_grad()
#with torch.no_grad():
# inputs = prepare_sequence(training_data[0][0], word_to_ix)
# tag_scores = model(inputs)
# print(tag_scores)
real_xy, guess_xy = [], []
lstm_state = torch.zeros([1,1,128]), torch.zeros([1,1,128])
for i in range(len(input_seq)):
if i+130 >= len(input_seq):
continue
inputseq = input_seq[i:i+128]
# Step 1. Remember that Pytorch accumulates gradients.
# We need to clear them out before each instance
model.zero_grad()
# Step 2. Get our inputs ready for the network, that is, turn them into
# Tensors of word indices.
sentence_in = torch.tensor(inputseq, dtype=torch.float)
# Step 3. Run our forward pass.
#print("MODEL INPUT: ", sentence_in.shape)
scout, lstm_state = model(sentence_in, lstm_state)
# Step 4. Compute the loss, gradients, and update the parameters by
# calling optimizer.step()
# loss = loss_function(tag_scores, targets)
#loss = loss_function(scout, torch.tensor(input_seq[i+129]))
loss = torch.abs(scout - input_seq[i+129])
print("DEBUG: LOSS=", loss)
loss.backward(retain_graph=True)
optimizer.step()
if i > 10240-1024:
real_xy.append(((i+129)/3, input_seq[i+129]))
guess_xy.append(((i+129)/3, scout[0,0,0].tolist()))
#print("REAL=", real_xy)
#print("GUESS=", guess_xy)
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm
x = np.arange(10)
ys = [i+x+(i*x)**2 for i in range(10)]
colors = cm.rainbow(np.linspace(0, 1, len(ys)))
for x, y in real_xy:
plt.scatter(x, y, color=colors[1])
for x, y in guess_xy:
plt.scatter(x, y, color=colors[6])
plt.show()
#for epoch in range(3): # again, normally you would NOT do 300 epochs, it is toy data
# for sentence, tags in training_data:
# # Step 1. Remember that Pytorch accumulates gradients.
# # We need to clear them out before each instance
# model.zero_grad()
#
# # Step 2. Get our inputs ready for the network, that is, turn them into
# # Tensors of word indices.
# sentence_in = prepare_sequence(sentence, word_to_ix)
# targets = prepare_sequence(tags, tag_to_ix)
#
# # Step 3. Run our forward pass.
# print("MODEL INPUT: ", sentence_in.shape)
# tag_scores = model(sentence_in)
#
# # Step 4. Compute the loss, gradients, and update the parameters by
# # calling optimizer.step()
# loss = loss_function(tag_scores, targets)
# loss.backward()
# optimizer.step()
# See what the scores are after training
#with torch.no_grad():
# inputs = prepare_sequence(training_data[0][0], word_to_ix)
# tag_scores = model(inputs)
#
# # The sentence is "the dog ate the apple". i,j corresponds to score for tag j
# # for word i. The predicted tag is the maximum scoring tag.
# # Here, we can see the predicted sequence below is 0 1 2 0 1
# # since 0 is index of the maximum value of row 1,
# # 1 is the index of maximum value of row 2, etc.
# # Which is DET NOUN VERB DET NOUN, the correct sequence!
# print(tag_scores)
......@@ -5,25 +5,38 @@
import json, time
from websocket import create_connection
import signal, sys, pickle
# [(Snapshot_Time, [(buy_price1, amount1), ...] , [(sell_price1, amount1), ...]), ...]
# [(1620457034392, [(56000, 0.01), (55900, 1), (55700, 30), ...] , [(57000, 0.01), (57100, 1), ...] ), (1620457034394, [...]), ...]
# The snapshots should has almost identical time-interval. Good for LSTM.
# Time axis: [history, older, newer, ..., latest]
realtime_shortterm_dataset_aggtrade = []
realtime_shortterm_dataset_aggtrade_size = 1024
realtime_shortterm_dataset_aggtrade_size = -1 #1024
# [(Trade_Time, PRICE, AMOUNT), ...]
# [(1620457034392, 56000, 0.5), (1620457034394, 56001, 0.05), ...]
# The trades usually have various time-interval. TODO: transform it to [(WeightedAvgPrice, Volume), ...] for every 1 minutes?
# Time axis: [history, older, newer, ..., latest]
realtime_shortterm_dataset_depth = []
realtime_shortterm_dataset_depth_size = 1024*1024
realtime_shortterm_dataset_depth_size = -1#1024*1024
# The trading thread would not start working, before finish analysing longterm dataset.
# Time-interval for longterm dataset is 1 minute.
longterm_dataset = []
def save_realtime_dataset_on_exit():
global realtime_shortterm_dataset_aggtrade, realtime_shortterm_dataset_depth
print("Saving {}+{} elements...".format(len(realtime_shortterm_dataset_aggtrade), len(realtime_shortterm_dataset_depth)))
pickle.dump((realtime_shortterm_dataset_aggtrade, realtime_shortterm_dataset_depth), open( "realtime_dataset_dump.pyobj", "wb"))
def load_realtime_dataset_on_start():
global realtime_shortterm_dataset_aggtrade, realtime_shortterm_dataset_depth
try:
realtime_shortterm_dataset_aggtrade, realtime_shortterm_dataset_depth = pickle.load(open( "realtime_dataset_dump.pyobj", "rb"))
print("Loaded {}+{} elements. ".format(len(realtime_shortterm_dataset_aggtrade), len(realtime_shortterm_dataset_depth)))
except:
print("No data to load. Skip data loading... ")
def dataset_maintain_thread_main():
global realtime_shortterm_dataset_aggtrade, realtime_shortterm_dataset_aggtrade_size, realtime_shortterm_dataset_depth, realtime_shortterm_dataset_depth_size
......@@ -47,7 +60,7 @@ def dataset_maintain_thread_main():
# Is a trade message
trade_time, price, amount = parsed.get('T'), parsed.get('p'), parsed.get('q')
realtime_shortterm_dataset_aggtrade.append((trade_time, price, amount))
if len(realtime_shortterm_dataset_aggtrade) > realtime_shortterm_dataset_aggtrade_size:
if realtime_shortterm_dataset_aggtrade_size != -1 and len(realtime_shortterm_dataset_aggtrade) > realtime_shortterm_dataset_aggtrade_size:
del realtime_shortterm_dataset_aggtrade[0]
elif 'b' in parsed.keys():
# Is a depth snapshot update
......@@ -55,7 +68,7 @@ def dataset_maintain_thread_main():
buy_makes_parsed = [(float(ele[0]), float(ele[1])) for ele in buy_makers]
sell_makes_parsed = [(float(ele[0]), float(ele[1])) for ele in sell_makers]
realtime_shortterm_dataset_depth.append((snapshot_time, buy_makes_parsed, sell_makes_parsed))
if len(realtime_shortterm_dataset_depth) > realtime_shortterm_dataset_depth_size:
if realtime_shortterm_dataset_depth_size != -1 and len(realtime_shortterm_dataset_depth) > realtime_shortterm_dataset_depth_size:
del realtime_shortterm_dataset_depth[0]
else:
print("ERROR: unexpected message (maybe ping message): ", message)
......@@ -71,5 +84,12 @@ def dataset_maintain_thread_main():
ws.close()
def sigint_handler(sig, frame):
print('Got SIGINT! Saving realtime dataset... ')
save_realtime_dataset_on_exit()
sys.exit(0)
signal.signal(signal.SIGINT, sigint_handler)
load_realtime_dataset_on_start()
dataset_maintain_thread_main()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment