Project1/HdfUtility.py at master · Msfund/Project1 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
import numpy as np
import pandas as pd
from pandas.io.pytables import HDFStore
import re
from dataUlt import *
import h5py
pd.set_option('io.hdf.default_format','table')
'''
HDF
    /Rawdata
        /CFE
            /IC
                /1m
                /5m
                /30m
                /1h
                /1d
    /Stitch
        /CFE
            /IF
               /Rule
                  /00
                  /01
               /Period
                  /1m
                  /5m
                  /15m
                  /60m
                  /1d
    /Indicator
        /CFE
            /IF
                /Indicator_name
'''
class HdfUtility:

    def hdfRead(self,path,excode,symbol,kind1,kind2,kind3,startdate=EXT_Start,enddate=EXT_End):
        # kind1为 'Rawdata',Stitch','Indicator'
        # kind2为 '00' '01'
        # kind3为 '1d' '60m' '30m' '15m' '5m' '1m'
        # 读各个频率的Rawdata: kind1='Rawdata',kind2=None,kind3='1d'
        # 读StitchRule:       kind1='Stitch', kind2='00',kind3=None
        # 读STitchData:       kind1='Stitch', kind2='00',kind3='1d'
        # 读Indicator：       kind1='Indicator',kind2='Indicator_name',kind3=None
        store = HDFStore(path,mode = 'r')
        if kind1 == EXT_Rawdata:
            key = kind1+'/'+excode+'/'+symbol+'/'+kind3
        elif kind1 == EXT_Stitch:
            key = kind1+'/'+excode+'/'+symbol+'/'+EXT_Rule+'/'+kind2 if kind3 == None else kind1+'/'+excode+'/'+symbol+'/'+EXT_Period+'/'+kind3+'/'+kind2
        elif kind1 == EXT_Indicator:
            key = kind1+'/'+excode+'/'+symbol+kind2
        else:
            print("kind not supported")
            return
        data = store[key].ix[((store[key].index.get_level_values(0)>=pd.to_datetime(startdate))&(store[key].index.get_level_values(0)<=pd.to_datetime(enddate))),:]
        store.close()
        if kind1 == EXT_Indicator:
            f = h5py.File(path,'r')
            params = f[key].attrs['Params']
            f.close()
            return data,params
        return data

    def hdfWrite(self,path,excode,symbol,indata,kind1,kind2,kind3):
        # kind1为 'Rawdata'、'Stitch'、'Indicator'
        # kind2为 '00' '01'
        # kind3为 '1d' '60m' '30m' '15m' '5m' '1m'
        # 写各个频率的Rawdata: kind1='Rawdata',kind2=None,kind3='1d'
        # 写StitchRule:       kind1='Stitch', kind2='00',kind3=None
        # 写StitchData:       kind1='Stitch', kind2='00',kind3='1d'
        # 写Indicator：       kind1='Indicator',kind2='Indicator_name',kind3='params'
        store = HDFStore(path,mode='a')
        if kind1 == EXT_Rawdata:
            key = kind1+'/'+excode+'/'+symbol+'/'+kind3
        elif kind1 == EXT_Stitch:
            key = kind1+'/'+excode+'/'+symbol+'/'+EXT_Rule+'/'+kind2 if kind3 == None else kind1+'/'+excode+'/'+symbol+'/'+EXT_Period+'/'+kind3+'/'+kind2
        elif kind1 == EXT_Indicator:
            key = kind1+'/'+excode+'/'+symbol+kind2
        else:
            print("kind not supported")
            return

        if kind1 ==EXT_Indicator:
            f = h5py.File(path,'a')
            try:
                store[key]
            except KeyError: # 路径不存在时创建
                store[key] = indata
                f[key].attrs['Params'] = kind3
            else:
                if f[key].attrs['Params'] == kind3: #Params匹配时合并
                    adddata = indata[~indata.index.isin(store[key].index)]
                    store.append(key,adddata)
                else: # Params不匹配时覆盖
                    store[key] = indata
                    f[key].attrs['Params'] = kind3
            f.close()
            store.close()
        else:
            try:
                store[key]
            except KeyError:
                store[key] = indata
            else:
                adddata = indata[~indata.index.isin(store[key].index)]
            if kind2 in [EXT_Series_00,EXT_Series_01]:
                adddata[EXT_Out_AdjFactor] = adddata[EXT_Out_AdjFactor]*store[key][EXT_Out_AdjFactor].iloc[-1]/adddata[EXT_Out_AdjFactor].iloc[0]
            store.append(key,adddata)
            store.close()