dataprovider.py 3.0 KB
Newer Older
C
chengxingyi 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from paddle.trainer.PyDataProvider2 import *
import sys
import numpy as np
TERM_NUM = 24
FORECASTING_NUM = 25
LABEL_VALUE_NUM = 4
Y
Yu Yang 已提交
21 22


C
chengxingyi 已提交
23 24 25 26 27 28 29 30 31
def initHook(settings, file_list, **kwargs):
    """
    Init hook is invoked before process data. It will set obj.slots and store data meta.

    :param settings: global object. It will passed to process routine.
    :type obj: object
    :param file_list: the meta file object, which passed from trainer_config.py,but unused in this function.
    :param kwargs: unused other arguments.
    """
Y
Yu Yang 已提交
32 33
    del kwargs  #unused 

C
chengxingyi 已提交
34 35 36 37 38 39 40 41 42
    settings.pool_size = sys.maxint
    #Use a time seires of the past as feature.
    #Dense_vector's expression form is [float,float,...,float]
    settings.slots = [dense_vector(TERM_NUM)]
    #There are next FORECASTING_NUM fragments you need predict.
    #Every predicted condition at time point has four states.
    for i in range(FORECASTING_NUM):
        settings.slots.append(integer_value(LABEL_VALUE_NUM))

Y
Yu Yang 已提交
43 44 45

@provider(
    init_hook=initHook, cache=CacheType.CACHE_PASS_IN_MEM, should_shuffle=True)
C
chengxingyi 已提交
46 47 48 49
def process(settings, file_name):
    with open(file_name) as f:
        #abandon fields name
        f.next()
Y
Yu Yang 已提交
50 51
        for row_num, line in enumerate(f):
            speeds = map(int, line.rstrip('\r\n').split(",")[1:])
C
chengxingyi 已提交
52 53 54
            # Get the max index.
            end_time = len(speeds)
            # Scanning and generating samples
Y
Yu Yang 已提交
55
            for i in range(TERM_NUM, end_time - FORECASTING_NUM):
C
chengxingyi 已提交
56
                # For dense slot
Y
Yu Yang 已提交
57
                pre_spd = map(float, speeds[i - TERM_NUM:i])
C
chengxingyi 已提交
58 59

                # Integer value need predicting, values start from 0, so every one minus 1.
Y
Yu Yang 已提交
60 61
                fol_spd = [i - 1 for i in speeds[i:i + FORECASTING_NUM]]

C
chengxingyi 已提交
62 63 64 65 66
                # Predicting label is missing, abandon the sample.
                if -1 in fol_spd:
                    continue
                yield [pre_spd] + fol_spd

Y
Yu Yang 已提交
67

C
chengxingyi 已提交
68 69 70 71
def predict_initHook(settings, file_list, **kwargs):
    settings.pool_size = sys.maxint
    settings.slots = [dense_vector(TERM_NUM)]

Y
Yu Yang 已提交
72 73

@provider(init_hook=predict_initHook, should_shuffle=False)
C
chengxingyi 已提交
74 75 76 77 78
def process_predict(settings, file_name):
    with open(file_name) as f:
        #abandon fields name
        f.next()
        for row_num, line in enumerate(f):
Y
Yu Yang 已提交
79
            speeds = map(int, line.rstrip('\r\n').split(","))
C
chengxingyi 已提交
80
            end_time = len(speeds)
Y
Yu Yang 已提交
81
            pre_spd = map(float, speeds[end_time - TERM_NUM:end_time])
C
chengxingyi 已提交
82
            yield pre_spd