# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os, random, csv, datetime, json import pandas as pd import numpy as np import argparse import logging import time logging.basicConfig( format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger("preprocess") logger.setLevel(logging.INFO) TRAIN_QUERIES_PATH = "./data_set_phase1/train_queries.csv" TRAIN_PLANS_PATH = "./data_set_phase1/train_plans.csv" TRAIN_CLICK_PATH = "./data_set_phase1/train_clicks.csv" PROFILES_PATH = "./data_set_phase1/profiles.csv" OUT_DIR = "./out" ORI_TRAIN_PATH = "train.txt" NORM_TRAIN_PATH = "normed_train.txt" #variable to control the ratio of positive and negative instance of transmode 0 which is original label of no click THRESHOLD_LABEL = 0.5 O1_MIN = 115.47 O1_MAX = 117.29 O2_MIN = 39.46 O2_MAX = 40.97 D1_MIN = 115.44 D1_MAX = 117.37 D2_MIN = 39.46 D2_MAX = 40.96 DISTANCE_MIN = 1.0 DISTANCE_MAX = 225864.0 THRESHOLD_DIS = 200000.0 PRICE_MIN = 200.0 PRICE_MAX = 92300.0 THRESHOLD_PRICE = 20000 ETA_MIN = 1.0 ETA_MAX = 72992.0 THRESHOLD_ETA = 10800.0 def build_norm_feature(): with open(os.path.join(OUT_DIR, NORM_TRAIN_PATH), 'w') as nf: with open(os.path.join(OUT_DIR, ORI_TRAIN_PATH), 'r') as f: for line in f: cur_map = json.loads(line) cur_map["plan"]["distance"] = (cur_map["plan"]["distance"] - DISTANCE_MIN) / (DISTANCE_MAX - DISTANCE_MIN) if cur_map["plan"]["price"]: cur_map["plan"]["price"] = (cur_map["plan"]["price"] - PRICE_MIN) / (PRICE_MAX - PRICE_MIN) else: cur_map["plan"]["price"] = 0.0 cur_map["plan"]["eta"] = (cur_map["plan"]["eta"] - ETA_MIN) / (ETA_MAX - ETA_MIN) cur_json_instance = json.dumps(cur_map) nf.write(cur_json_instance + '\n') def preprocess(): """ Construct the train data indexed by session id and mode id jointly. Convert all the raw features (user profile, od pair, req time, click time, eta, price, distance, transport mode) to one-hot ids used for embedding. We split the one-hot features into two categories: user feature and context feature for better understanding of FM algorithm. Note that the user profile is already provided by one-hot encoded form, we treat it as embedded vector for unity with the context feature and easily using of PaddlePaddle embedding layer. Given the train clicks data, we label each train instance with 1 or 0 depend on if this instance is clicked or not include non-click case. To Be Changed :return: """ train_data_dict = {} with open("./weather.json", 'r') as f: weather_dict = json.load(f) with open(TRAIN_QUERIES_PATH, 'r') as f: csv_reader = csv.reader(f, delimiter=',') train_index_list = [] for k, line in enumerate(csv_reader): if k == 0: continue if line[0] == "": continue if line[1] == "": train_index_list.append(line[0] + "_0") else: train_index_list.append(line[0] + "_" + line[1]) train_index = line[0] train_data_dict[train_index] = {} train_data_dict[train_index]["pid"] = line[1] train_data_dict[train_index]["query"] = {} train_data_dict[train_index]["weather"] = {} reqweekday = datetime.datetime.strptime(line[2], '%Y-%m-%d %H:%M:%S').strftime("%w") reqhour = datetime.datetime.strptime(line[2], '%Y-%m-%d %H:%M:%S').strftime("%H") # weather related features, no big use, maybe more detailed weather information is better date_key = datetime.datetime.strptime(line[2], '%Y-%m-%d %H:%M:%S').strftime("%m-%d") train_data_dict[train_index]["weather"] = {} train_data_dict[train_index]["weather"].update({"max_temp": weather_dict[date_key]["max_temp"]}) train_data_dict[train_index]["weather"].update({"min_temp": weather_dict[date_key]["min_temp"]}) train_data_dict[train_index]["weather"].update({"wea": weather_dict[date_key]["weather"]}) train_data_dict[train_index]["weather"].update({"wind": weather_dict[date_key]["wind"]}) train_data_dict[train_index]["query"].update({"weekday":reqweekday}) train_data_dict[train_index]["query"].update({"hour":reqhour}) o = line[3].split(',') o_first = o[0] o_second = o[1] train_data_dict[train_index]["query"].update({"o1":float(o_first)}) train_data_dict[train_index]["query"].update({"o2":float(o_second)}) d = line[4].split(',') d_first = d[0] d_second = d[1] train_data_dict[train_index]["query"].update({"d1":float(d_first)}) train_data_dict[train_index]["query"].update({"d2":float(d_second)}) plan_map = {} plan_data = pd.read_csv(TRAIN_PLANS_PATH) for index, row in plan_data.iterrows(): plans_str = row['plans'] plans_list = json.loads(plans_str) session_id = str(row['sid']) # train_data_dict[session_id]["plans"] = [] plan_map[session_id] = plans_list profile_map = {} with open(PROFILES_PATH, 'r') as f: csv_reader = csv.reader(f, delimiter=',') for k, line in enumerate(csv_reader): if k == 0: continue profile_map[line[0]] = [i for i in range(len(line)) if line[i] == "1.0"] session_click_map = {} with open(TRAIN_CLICK_PATH, 'r') as f: csv_reader = csv.reader(f, delimiter=',') for k, line in enumerate(csv_reader): if k == 0: continue if line[0] == "" or line[1] == "" or line[2] == "": continue session_click_map[line[0]] = line[2] #return train_data_dict, profile_map, session_click_map, plan_map generate_sparse_features(train_data_dict, profile_map, session_click_map, plan_map) def generate_sparse_features(train_data_dict, profile_map, session_click_map, plan_map): if not os.path.isdir(OUT_DIR): os.mkdir(OUT_DIR) with open(os.path.join(OUT_DIR, ORI_TRAIN_PATH), 'w') as f_train: for session_id, plan_list in plan_map.items(): if session_id not in train_data_dict: continue cur_map = train_data_dict[session_id] if cur_map["pid"] != "": cur_map["profile"] = profile_map[cur_map["pid"]] else: cur_map["profile"] = [0] #rank information related feature whole_rank = 0 for plan in plan_list: whole_rank += 1 cur_map["mode_rank" + str(whole_rank)] = plan["transport_mode"] if whole_rank < 5: for r in range(whole_rank + 1, 6): cur_map["mode_rank" + str(r)] = -1 cur_map["whole_rank"] = whole_rank flag_click = False rank = 1 price_list = [] eta_list = [] distance_list = [] for plan in plan_list: if not plan["price"]: price_list.append(0) else: price_list.append(int(plan["price"])) eta_list.append(int(plan["eta"])) distance_list.append(int(plan["distance"])) price_list.sort(reverse=False) eta_list.sort(reverse=False) distance_list.sort(reverse=False) for plan in plan_list: if plan["price"] and int(plan["price"]) == price_list[0]: cur_map["mode_min_price"] = plan["transport_mode"] if plan["price"] and int(plan["price"]) == price_list[-1]: cur_map["mode_max_price"] = plan["transport_mode"] if int(plan["eta"]) == eta_list[0]: cur_map["mode_min_eta"] = plan["transport_mode"] if int(plan["eta"]) == eta_list[-1]: cur_map["mode_max_eta"] = plan["transport_mode"] if int(plan["distance"]) == distance_list[0]: cur_map["mode_min_distance"] = plan["transport_mode"] if int(plan["distance"]) == distance_list[-1]: cur_map["mode_max_distance"] = plan["transport_mode"] if "mode_min_price" not in cur_map: cur_map["mode_min_price"] = -1 if "mode_max_price" not in cur_map: cur_map["mode_max_price"] = -1 for plan in plan_list: if ("transport_mode" in plan) and (session_id in session_click_map) and ( int(plan["transport_mode"]) == int(session_click_map[session_id])): flag_click = True if flag_click: for plan in plan_list: cur_price = int(plan["price"]) if plan["price"] else 0 cur_eta = int(plan["eta"]) cur_distance = int(plan["distance"]) cur_map["price_rank"] = price_list.index(cur_price) + 1 cur_map["eta_rank"] = eta_list.index(cur_eta) + 1 cur_map["distance_rank"] = distance_list.index(cur_distance) + 1 if ("transport_mode" in plan) and (session_id in session_click_map) and ( int(plan["transport_mode"]) == int(session_click_map[session_id])): cur_map["plan"] = plan cur_map["label"] = 1 else: cur_map["plan"] = plan cur_map["label"] = 0 cur_map["plan_rank"] = rank rank += 1 cur_json_instance = json.dumps(cur_map) f_train.write(cur_json_instance + '\n') cur_map["plan"] = {} #since we define a new ctr task from original task, we use a basic way to generate instances of transport mode 0. #There should be a optimal strategy to generate instances of transport mode 0 if not flag_click: cur_map["plan"]["distance"] = -1 cur_map["plan"]["price"] = -1 cur_map["plan"]["eta"] = -1 cur_map["plan"]["transport_mode"] = 0 cur_map["plan_rank"] = 0 cur_map["price_rank"] = 0 cur_map["eta_rank"] = 0 cur_map["distance_rank"] = 0 cur_map["label"] = 1 cur_json_instance = json.dumps(cur_map) f_train.write(cur_json_instance + '\n') else: if random.random() < THRESHOLD_LABEL: cur_map["plan"]["distance"] = -1 cur_map["plan"]["price"] = -1 cur_map["plan"]["eta"] = -1 cur_map["plan"]["transport_mode"] = 0 cur_map["plan_rank"] = 0 cur_map["price_rank"] = 0 cur_map["eta_rank"] = 0 cur_map["distance_rank"] = 0 cur_map["label"] = 0 cur_json_instance = json.dumps(cur_map) f_train.write(cur_json_instance + '\n') build_norm_feature() if __name__ == "__main__": preprocess()