pre_process_test.py

# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os, sys, time, random, csv, datetime, json
import pandas as pd
import numpy as np
import argparse
import logging
import time

logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger("preprocess")
logger.setLevel(logging.INFO)

TEST_QUERIES_PATH = "./data_set_phase1/test_queries.csv"
TEST_PLANS_PATH = "./data_set_phase1/test_plans.csv"
TRAIN_CLICK_PATH = "./data_set_phase1/train_clicks.csv"
PROFILES_PATH = "./data_set_phase1/profiles.csv"
OUT_NORM_TEST_PATH = "./out/normed_test_session.txt"
OUT_RAW_TEST_PATH = "./out/test_session.txt"

O1_MIN = 115.47
O1_MAX = 117.29

O2_MIN = 39.46
O2_MAX = 40.97

D1_MIN = 115.44
D1_MAX = 117.37

D2_MIN = 39.46
D2_MAX = 40.96
SCALE_OD = 0.02

DISTANCE_MIN = 1.0
DISTANCE_MAX = 225864.0
THRESHOLD_DIS = 40000.0
SCALE_DIS = 500

PRICE_MIN = 200.0
PRICE_MAX = 92300.0
THRESHOLD_PRICE = 20000
SCALE_PRICE = 100

ETA_MIN = 1.0
ETA_MAX = 72992.0
THRESHOLD_ETA = 10800.0
SCALE_ETA = 120


def build_norm_feature():
    with open(OUT_NORM_TEST_PATH, 'w') as nf:
        with open(OUT_RAW_TEST_PATH, 'r') as f:
            for line in f:
                cur_map = json.loads(line)

                if cur_map["plan"]["distance"] > THRESHOLD_DIS:
                    cur_map["plan"]["distance"] = int(THRESHOLD_DIS)
                elif cur_map["plan"]["distance"] > 0:
                    cur_map["plan"]["distance"] = int(cur_map["plan"]["distance"] / SCALE_DIS)

                if cur_map["plan"]["price"] and cur_map["plan"]["price"] > THRESHOLD_PRICE:
                    cur_map["plan"]["price"] = int(THRESHOLD_PRICE)
                elif not cur_map["plan"]["price"] or cur_map["plan"]["price"] < 0:
                    cur_map["plan"]["price"] = 0
                else:
                    cur_map["plan"]["price"] = int(cur_map["plan"]["price"] / SCALE_PRICE)

                if cur_map["plan"]["eta"] > THRESHOLD_ETA:
                    cur_map["plan"]["eta"] = int(THRESHOLD_ETA)
                elif cur_map["plan"]["eta"] > 0:
                    cur_map["plan"]["eta"] = int(cur_map["plan"]["eta"] / SCALE_ETA)

                # o1
                if cur_map["query"]["o1"] > O1_MAX:
                    cur_map["query"]["o1"] = int((O1_MAX - O1_MIN) / SCALE_OD + 1)
                elif cur_map["query"]["o1"] < O1_MIN:
                    cur_map["query"]["o1"] = 0
                else:
                    cur_map["query"]["o1"] = int((cur_map["query"]["o1"] - O1_MIN) / 0.02)

                # o2
                if cur_map["query"]["o2"] > O2_MAX:
                    cur_map["query"]["o2"] = int((O2_MAX - O2_MIN) / SCALE_OD + 1)
                elif cur_map["query"]["o2"] < O2_MIN:
                    cur_map["query"]["o2"] = 0
                else:
                    cur_map["query"]["o2"] = int((cur_map["query"]["o2"] - O2_MIN) / 0.02)

                # d1
                if cur_map["query"]["d1"] > D1_MAX:
                    cur_map["query"]["d1"] = int((D1_MAX - D1_MIN) / SCALE_OD + 1)
                elif cur_map["query"]["d1"] < D1_MIN:
                    cur_map["query"]["d1"] = 0
                else:
                    cur_map["query"]["d1"] = int((cur_map["query"]["d1"] - D1_MIN) / SCALE_OD)

                # d2
                if cur_map["query"]["d2"] > D2_MAX:
                    cur_map["query"]["d2"] = int((D2_MAX - D2_MIN) / SCALE_OD + 1)
                elif cur_map["query"]["d2"] < D2_MIN:
                    cur_map["query"]["d2"] = 0
                else:
                    cur_map["query"]["d2"] = int((cur_map["query"]["d2"] - D2_MIN) / SCALE_OD)

                cur_json_instance = json.dumps(cur_map)
                nf.write(cur_json_instance + '\n')


def preprocess():
    """
    Construct the train data indexed by session id and mode id jointly. Convert some of the raw features (user profile,
    od pair, req time, click time, eta, price, distance, transport mode) to one-hot ids used for
    embedding. We split the one-hot features into two categories: user feature and context feature for
    better understanding of FM algorithm.
    Note that the user profile is already provided by one-hot encoded form, we convert it back to the
    ids for unity with the context feature and easily using of PaddlePaddle embedding layer. Given the
    train clicks data, we label each train instance with 1 or 0 depend on if this instance is clicked or
    not.
    :return:
    """

    train_data_dict = {}
    with open("./weather.json", 'r') as f:
        weather_dict = json.load(f)

    with open(TEST_QUERIES_PATH, 'r') as f:
        csv_reader = csv.reader(f, delimiter=',')
        train_index_list = []
        for k, line in enumerate(csv_reader):
            if k == 0: continue
            if line[0] == "": continue
            if line[1] == "":
                train_index_list.append(line[0] + "_0")
            else:
                train_index_list.append(line[0] + "_" + line[1])

            train_index = line[0]
            train_data_dict[train_index] = {}
            train_data_dict[train_index]["pid"] = line[1]
            train_data_dict[train_index]["query"] = {}

            reqweekday = datetime.datetime.strptime(line[2], '%Y-%m-%d %H:%M:%S').strftime("%w")
            reqhour = datetime.datetime.strptime(line[2], '%Y-%m-%d %H:%M:%S').strftime("%H")

            date_key = datetime.datetime.strptime(line[2], '%Y-%m-%d %H:%M:%S').strftime("%m-%d")
            train_data_dict[train_index]["weather"] = {}
            train_data_dict[train_index]["weather"].update({"max_temp": weather_dict[date_key]["max_temp"]})
            train_data_dict[train_index]["weather"].update({"min_temp": weather_dict[date_key]["min_temp"]})
            train_data_dict[train_index]["weather"].update({"wea": weather_dict[date_key]["weather"]})
            train_data_dict[train_index]["weather"].update({"wind": weather_dict[date_key]["wind"]})

            train_data_dict[train_index]["query"].update({"weekday":reqweekday})
            train_data_dict[train_index]["query"].update({"hour":reqhour})

            o = line[3].split(',')
            o_first = o[0]
            o_second = o[1]
            train_data_dict[train_index]["query"].update({"o1":float(o_first)})
            train_data_dict[train_index]["query"].update({"o2":float(o_second)})

            d = line[4].split(',')
            d_first = d[0]
            d_second = d[1]
            train_data_dict[train_index]["query"].update({"d1":float(d_first)})
            train_data_dict[train_index]["query"].update({"d2":float(d_second)})

    plan_map = {}
    plan_data = pd.read_csv(TEST_PLANS_PATH)
    for index, row in plan_data.iterrows():
        plans_str = row['plans']
        plans_list = json.loads(plans_str)
        session_id = str(row['sid'])
        # train_data_dict[session_id]["plans"] = []
        plan_map[session_id] = plans_list

    profile_map = {}
    with open(PROFILES_PATH, 'r') as f:
        csv_reader = csv.reader(f, delimiter=',')
        for k, line in enumerate(csv_reader):
            if k == 0: continue
            profile_map[line[0]] = [i for i in range(len(line)) if line[i] == "1.0"]

    session_click_map = {}
    with open(TRAIN_CLICK_PATH, 'r') as f:
        csv_reader = csv.reader(f, delimiter=',')
        for k, line in enumerate(csv_reader):
            if k == 0: continue
            if line[0] == "" or line[1] == "" or line[2] == "":
                continue
            session_click_map[line[0]] = line[2]
    #return train_data_dict, profile_map, session_click_map, plan_map
    generate_sparse_features(train_data_dict, profile_map, session_click_map, plan_map)


def generate_sparse_features(train_data_dict, profile_map, session_click_map, plan_map):
    if not os.path.isdir("./out/"):
        os.mkdir("./out/")
    with open(os.path.join("./out/", "test_session.txt"), 'w') as f_train:
        for session_id, plan_list in plan_map.items():
            if session_id not in train_data_dict:
                continue
            cur_map = train_data_dict[session_id]
            cur_map["session_id"] = session_id
            if cur_map["pid"] != "":
                cur_map["profile"] = profile_map[cur_map["pid"]]
            else:
                cur_map["profile"] = [0]
            del cur_map["pid"]
            whole_rank = 0
            for plan in plan_list:
                whole_rank += 1
                cur_map["mode_rank" + str(whole_rank)] = plan["transport_mode"]

            if whole_rank < 5:
                for r in range(whole_rank + 1, 6):
                    cur_map["mode_rank" + str(r)] = -1

            cur_map["whole_rank"] = whole_rank
            flag_click = False
            rank = 1

            price_list = []
            eta_list = []
            distance_list = []
            for plan in plan_list:
                if not plan["price"]:
                    price_list.append(0)
                else:
                    price_list.append(int(plan["price"]))
                eta_list.append(int(plan["eta"]))
                distance_list.append(int(plan["distance"]))
            price_list.sort(reverse=False)
            eta_list.sort(reverse=False)
            distance_list.sort(reverse=False)

            for plan in plan_list:
                if plan["price"] and int(plan["price"]) == price_list[0]:
                    cur_map["mode_min_price"] = plan["transport_mode"]
                if plan["price"] and int(plan["price"]) == price_list[-1]:
                    cur_map["mode_max_price"] = plan["transport_mode"]
                if int(plan["eta"]) == eta_list[0]:
                    cur_map["mode_min_eta"] = plan["transport_mode"]
                if int(plan["eta"]) == eta_list[-1]:
                    cur_map["mode_max_eta"] = plan["transport_mode"]
                if int(plan["distance"]) == distance_list[0]:
                    cur_map["mode_min_distance"] = plan["transport_mode"]
                if int(plan["distance"]) == distance_list[-1]:
                    cur_map["mode_max_distance"] = plan["transport_mode"]
            if "mode_min_price" not in cur_map:
                cur_map["mode_min_price"] = -1
            if "mode_max_price" not in cur_map:
                cur_map["mode_max_price"] = -1


            for plan in plan_list:
                cur_price = int(plan["price"]) if plan["price"] else 0
                cur_eta = int(plan["eta"])
                cur_distance = int(plan["distance"])
                cur_map["price_rank"] = price_list.index(cur_price) + 1
                cur_map["eta_rank"] = eta_list.index(cur_eta) + 1
                cur_map["distance_rank"] = distance_list.index(cur_distance) + 1

                if ("transport_mode" in plan) and (session_id in session_click_map) and (
                        int(plan["transport_mode"]) == int(session_click_map[session_id])):
                    cur_map["plan"] = plan
                    cur_map["label"] = 1
                    flag_click = True
                    # print("label is 1")
                else:
                    cur_map["plan"] = plan
                    cur_map["label"] = 0

                cur_map["plan_rank"] = rank
                rank += 1
                cur_json_instance = json.dumps(cur_map)
                f_train.write(cur_json_instance + '\n')

            cur_map["plan"]["distance"] = -1
            cur_map["plan"]["price"] = -1
            cur_map["plan"]["eta"] = -1
            cur_map["plan"]["transport_mode"] = 0
            cur_map["plan_rank"] = 0
            cur_map["price_rank"] = 0
            cur_map["eta_rank"] = 0
            cur_map["plan_rank"] = 0
            cur_map["label"] = 1
            cur_json_instance = json.dumps(cur_map)
            f_train.write(cur_json_instance + '\n')

    build_norm_feature()


if __name__ == "__main__":
    preprocess()