From e8f2dd86434799d6bd907ef67b6cd9b47c186df5 Mon Sep 17 00:00:00 2001 From: kechxu Date: Thu, 31 Jan 2019 16:50:43 -0800 Subject: [PATCH] tools/prediction: reorgnize folders --- .../common/__init__.py | 0 .../common/bounding_rectangle.py | 0 .../common/configure.py | 0 .../common/data_preprocess.py | 0 .../common/feature_io.py | 0 .../common/log.py | 0 .../common/online_to_offline.py | 2 +- .../common/rotation2d.py | 0 .../common/trajectory.py | 0 .../common/util.py | 0 .../common/vector2d.py | 0 .../cruiseMLP_train.py | 0 .../cruise_h5_preprocessing.py | 0 .../cruise_models.py | 0 .../combine_features_and_labels.py | 0 ...ombine_features_and_labels_for_junction.py | 0 .../features_labels_utils.py | 0 .../generate_cruise_labels.py | 17 +--- .../generate_future_trajectory.py | 5 +- .../generate_junction_labels.py | 5 +- .../data_preprocessing}/generate_labels.py | 5 +- .../data_preprocessing/merge_label_dicts.py | 0 .../junctionMLP_train.py | 0 .../{mlp_train => data_pipelines}/merge_h5.py | 0 .../mlp_train.py | 0 .../proto/__init__.py | 0 .../proto/cruise_model.proto | 0 .../proto/fnn_model.proto | 0 .../combine_features_and_labels_script.sh | 4 +- .../scripts/generate_labels.sh | 0 .../scripts/generate_model_feature.sh | 0 .../scripts}/merge_label_dicts_script.sh | 0 .../scripts/records_to_data_for_learning.sh | 0 .../scripts/records_to_dump_feature_proto.sh | 0 .../scripts/records_to_prediction_result.sh | 0 .../mlp_train/feature_extraction.sh | 75 --------------- .../mlp_train/generate_cruise_h5.py | 91 ------------------- .../mlp_train/generate_junction_h5.py | 87 ------------------ .../prediction/mlp_train/generate_output.py | 87 ------------------ .../scripts/generate_feature_proto.sh | 41 --------- 40 files changed, 14 insertions(+), 405 deletions(-) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/__init__.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/bounding_rectangle.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/configure.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/data_preprocess.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/feature_io.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/log.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/online_to_offline.py (99%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/rotation2d.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/trajectory.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/util.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/common/vector2d.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/cruiseMLP_train.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/cruise_h5_preprocessing.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/cruise_models.py (100%) rename modules/tools/prediction/{learning_algorithms => data_pipelines}/data_preprocessing/combine_features_and_labels.py (100%) rename modules/tools/prediction/{learning_algorithms => data_pipelines}/data_preprocessing/combine_features_and_labels_for_junction.py (100%) rename modules/tools/prediction/{learning_algorithms => data_pipelines}/data_preprocessing/features_labels_utils.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines/data_preprocessing}/generate_cruise_labels.py (75%) rename modules/tools/prediction/{mlp_train => data_pipelines/data_preprocessing}/generate_future_trajectory.py (91%) rename modules/tools/prediction/{mlp_train => data_pipelines/data_preprocessing}/generate_junction_labels.py (91%) rename modules/tools/prediction/{mlp_train => data_pipelines/data_preprocessing}/generate_labels.py (91%) rename modules/tools/prediction/{learning_algorithms => data_pipelines}/data_preprocessing/merge_label_dicts.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/junctionMLP_train.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/merge_h5.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/mlp_train.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/proto/__init__.py (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/proto/cruise_model.proto (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/proto/fnn_model.proto (100%) rename modules/tools/prediction/{learning_algorithms/data_preprocessing => data_pipelines/scripts}/combine_features_and_labels_script.sh (79%) rename modules/tools/prediction/{mlp_train => data_pipelines}/scripts/generate_labels.sh (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/scripts/generate_model_feature.sh (100%) rename modules/tools/prediction/{learning_algorithms/data_preprocessing => data_pipelines/scripts}/merge_label_dicts_script.sh (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/scripts/records_to_data_for_learning.sh (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/scripts/records_to_dump_feature_proto.sh (100%) rename modules/tools/prediction/{mlp_train => data_pipelines}/scripts/records_to_prediction_result.sh (100%) delete mode 100644 modules/tools/prediction/mlp_train/feature_extraction.sh delete mode 100644 modules/tools/prediction/mlp_train/generate_cruise_h5.py delete mode 100644 modules/tools/prediction/mlp_train/generate_junction_h5.py delete mode 100644 modules/tools/prediction/mlp_train/generate_output.py delete mode 100644 modules/tools/prediction/mlp_train/scripts/generate_feature_proto.sh diff --git a/modules/tools/prediction/mlp_train/common/__init__.py b/modules/tools/prediction/data_pipelines/common/__init__.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/__init__.py rename to modules/tools/prediction/data_pipelines/common/__init__.py diff --git a/modules/tools/prediction/mlp_train/common/bounding_rectangle.py b/modules/tools/prediction/data_pipelines/common/bounding_rectangle.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/bounding_rectangle.py rename to modules/tools/prediction/data_pipelines/common/bounding_rectangle.py diff --git a/modules/tools/prediction/mlp_train/common/configure.py b/modules/tools/prediction/data_pipelines/common/configure.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/configure.py rename to modules/tools/prediction/data_pipelines/common/configure.py diff --git a/modules/tools/prediction/mlp_train/common/data_preprocess.py b/modules/tools/prediction/data_pipelines/common/data_preprocess.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/data_preprocess.py rename to modules/tools/prediction/data_pipelines/common/data_preprocess.py diff --git a/modules/tools/prediction/mlp_train/common/feature_io.py b/modules/tools/prediction/data_pipelines/common/feature_io.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/feature_io.py rename to modules/tools/prediction/data_pipelines/common/feature_io.py diff --git a/modules/tools/prediction/mlp_train/common/log.py b/modules/tools/prediction/data_pipelines/common/log.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/log.py rename to modules/tools/prediction/data_pipelines/common/log.py diff --git a/modules/tools/prediction/mlp_train/common/online_to_offline.py b/modules/tools/prediction/data_pipelines/common/online_to_offline.py similarity index 99% rename from modules/tools/prediction/mlp_train/common/online_to_offline.py rename to modules/tools/prediction/data_pipelines/common/online_to_offline.py index f268f00f25..556db402d0 100644 --- a/modules/tools/prediction/mlp_train/common/online_to_offline.py +++ b/modules/tools/prediction/data_pipelines/common/online_to_offline.py @@ -25,7 +25,7 @@ from google.protobuf.internal import decoder from google.protobuf.internal import encoder from modules.prediction.proto import offline_features_pb2 -from common.bounding_rectangle import BoundingRectangle +from bounding_rectangle import BoundingRectangle from configure import parameters param_fea = parameters['feature'] diff --git a/modules/tools/prediction/mlp_train/common/rotation2d.py b/modules/tools/prediction/data_pipelines/common/rotation2d.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/rotation2d.py rename to modules/tools/prediction/data_pipelines/common/rotation2d.py diff --git a/modules/tools/prediction/mlp_train/common/trajectory.py b/modules/tools/prediction/data_pipelines/common/trajectory.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/trajectory.py rename to modules/tools/prediction/data_pipelines/common/trajectory.py diff --git a/modules/tools/prediction/mlp_train/common/util.py b/modules/tools/prediction/data_pipelines/common/util.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/util.py rename to modules/tools/prediction/data_pipelines/common/util.py diff --git a/modules/tools/prediction/mlp_train/common/vector2d.py b/modules/tools/prediction/data_pipelines/common/vector2d.py similarity index 100% rename from modules/tools/prediction/mlp_train/common/vector2d.py rename to modules/tools/prediction/data_pipelines/common/vector2d.py diff --git a/modules/tools/prediction/mlp_train/cruiseMLP_train.py b/modules/tools/prediction/data_pipelines/cruiseMLP_train.py similarity index 100% rename from modules/tools/prediction/mlp_train/cruiseMLP_train.py rename to modules/tools/prediction/data_pipelines/cruiseMLP_train.py diff --git a/modules/tools/prediction/mlp_train/cruise_h5_preprocessing.py b/modules/tools/prediction/data_pipelines/cruise_h5_preprocessing.py similarity index 100% rename from modules/tools/prediction/mlp_train/cruise_h5_preprocessing.py rename to modules/tools/prediction/data_pipelines/cruise_h5_preprocessing.py diff --git a/modules/tools/prediction/mlp_train/cruise_models.py b/modules/tools/prediction/data_pipelines/cruise_models.py similarity index 100% rename from modules/tools/prediction/mlp_train/cruise_models.py rename to modules/tools/prediction/data_pipelines/cruise_models.py diff --git a/modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels.py b/modules/tools/prediction/data_pipelines/data_preprocessing/combine_features_and_labels.py similarity index 100% rename from modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels.py rename to modules/tools/prediction/data_pipelines/data_preprocessing/combine_features_and_labels.py diff --git a/modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels_for_junction.py b/modules/tools/prediction/data_pipelines/data_preprocessing/combine_features_and_labels_for_junction.py similarity index 100% rename from modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels_for_junction.py rename to modules/tools/prediction/data_pipelines/data_preprocessing/combine_features_and_labels_for_junction.py diff --git a/modules/tools/prediction/learning_algorithms/data_preprocessing/features_labels_utils.py b/modules/tools/prediction/data_pipelines/data_preprocessing/features_labels_utils.py similarity index 100% rename from modules/tools/prediction/learning_algorithms/data_preprocessing/features_labels_utils.py rename to modules/tools/prediction/data_pipelines/data_preprocessing/features_labels_utils.py diff --git a/modules/tools/prediction/mlp_train/generate_cruise_labels.py b/modules/tools/prediction/data_pipelines/data_preprocessing/generate_cruise_labels.py similarity index 75% rename from modules/tools/prediction/mlp_train/generate_cruise_labels.py rename to modules/tools/prediction/data_pipelines/data_preprocessing/generate_cruise_labels.py index cfdb160f60..20d60b7a1b 100644 --- a/modules/tools/prediction/mlp_train/generate_cruise_labels.py +++ b/modules/tools/prediction/data_pipelines/data_preprocessing/generate_cruise_labels.py @@ -22,22 +22,9 @@ import glob import argparse import logging -from common.configure import parameters -from common.feature_io import load_protobuf -from common.feature_io import save_protobuf -from common.feature_io import build_trajectory -from common.trajectory import TrajectoryToSample -from common.online_to_offline import LabelGenerator +sys.path.append('/apollo/modules/tools/prediction/data_pipelines/common/') - -import os -import sys -import glob -import argparse -import logging - -from common.configure import parameters -from common.online_to_offline import LabelGenerator +from online_to_offline import LabelGenerator if __name__ == "__main__": diff --git a/modules/tools/prediction/mlp_train/generate_future_trajectory.py b/modules/tools/prediction/data_pipelines/data_preprocessing/generate_future_trajectory.py similarity index 91% rename from modules/tools/prediction/mlp_train/generate_future_trajectory.py rename to modules/tools/prediction/data_pipelines/data_preprocessing/generate_future_trajectory.py index 48b592b2fe..0e615618a5 100644 --- a/modules/tools/prediction/mlp_train/generate_future_trajectory.py +++ b/modules/tools/prediction/data_pipelines/data_preprocessing/generate_future_trajectory.py @@ -20,8 +20,9 @@ import glob import argparse import logging -from common.configure import parameters -from common.online_to_offline import LabelGenerator +sys.path.append('/apollo/modules/tools/prediction/data_pipelines/common/') + +from online_to_offline import LabelGenerator if __name__ == "__main__": diff --git a/modules/tools/prediction/mlp_train/generate_junction_labels.py b/modules/tools/prediction/data_pipelines/data_preprocessing/generate_junction_labels.py similarity index 91% rename from modules/tools/prediction/mlp_train/generate_junction_labels.py rename to modules/tools/prediction/data_pipelines/data_preprocessing/generate_junction_labels.py index 23330acbd8..a1e00c5f16 100644 --- a/modules/tools/prediction/mlp_train/generate_junction_labels.py +++ b/modules/tools/prediction/data_pipelines/data_preprocessing/generate_junction_labels.py @@ -20,8 +20,9 @@ import glob import argparse import logging -from common.configure import parameters -from common.online_to_offline import LabelGenerator +sys.path.append('/apollo/modules/tools/prediction/data_pipelines/common/') + +from online_to_offline import LabelGenerator if __name__ == "__main__": diff --git a/modules/tools/prediction/mlp_train/generate_labels.py b/modules/tools/prediction/data_pipelines/data_preprocessing/generate_labels.py similarity index 91% rename from modules/tools/prediction/mlp_train/generate_labels.py rename to modules/tools/prediction/data_pipelines/data_preprocessing/generate_labels.py index 9cccfa1b50..14edcb5f09 100644 --- a/modules/tools/prediction/mlp_train/generate_labels.py +++ b/modules/tools/prediction/data_pipelines/data_preprocessing/generate_labels.py @@ -20,8 +20,9 @@ import glob import argparse import logging -from common.configure import parameters -from common.online_to_offline import LabelGenerator +sys.path.append('/apollo/modules/tools/prediction/data_pipelines/common/') + +from online_to_offline import LabelGenerator if __name__ == "__main__": diff --git a/modules/tools/prediction/learning_algorithms/data_preprocessing/merge_label_dicts.py b/modules/tools/prediction/data_pipelines/data_preprocessing/merge_label_dicts.py similarity index 100% rename from modules/tools/prediction/learning_algorithms/data_preprocessing/merge_label_dicts.py rename to modules/tools/prediction/data_pipelines/data_preprocessing/merge_label_dicts.py diff --git a/modules/tools/prediction/mlp_train/junctionMLP_train.py b/modules/tools/prediction/data_pipelines/junctionMLP_train.py similarity index 100% rename from modules/tools/prediction/mlp_train/junctionMLP_train.py rename to modules/tools/prediction/data_pipelines/junctionMLP_train.py diff --git a/modules/tools/prediction/mlp_train/merge_h5.py b/modules/tools/prediction/data_pipelines/merge_h5.py similarity index 100% rename from modules/tools/prediction/mlp_train/merge_h5.py rename to modules/tools/prediction/data_pipelines/merge_h5.py diff --git a/modules/tools/prediction/mlp_train/mlp_train.py b/modules/tools/prediction/data_pipelines/mlp_train.py similarity index 100% rename from modules/tools/prediction/mlp_train/mlp_train.py rename to modules/tools/prediction/data_pipelines/mlp_train.py diff --git a/modules/tools/prediction/mlp_train/proto/__init__.py b/modules/tools/prediction/data_pipelines/proto/__init__.py similarity index 100% rename from modules/tools/prediction/mlp_train/proto/__init__.py rename to modules/tools/prediction/data_pipelines/proto/__init__.py diff --git a/modules/tools/prediction/mlp_train/proto/cruise_model.proto b/modules/tools/prediction/data_pipelines/proto/cruise_model.proto similarity index 100% rename from modules/tools/prediction/mlp_train/proto/cruise_model.proto rename to modules/tools/prediction/data_pipelines/proto/cruise_model.proto diff --git a/modules/tools/prediction/mlp_train/proto/fnn_model.proto b/modules/tools/prediction/data_pipelines/proto/fnn_model.proto similarity index 100% rename from modules/tools/prediction/mlp_train/proto/fnn_model.proto rename to modules/tools/prediction/data_pipelines/proto/fnn_model.proto diff --git a/modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels_script.sh b/modules/tools/prediction/data_pipelines/scripts/combine_features_and_labels_script.sh similarity index 79% rename from modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels_script.sh rename to modules/tools/prediction/data_pipelines/scripts/combine_features_and_labels_script.sh index e0c72e325a..1d291c0d09 100644 --- a/modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels_script.sh +++ b/modules/tools/prediction/data_pipelines/scripts/combine_features_and_labels_script.sh @@ -31,7 +31,7 @@ source /apollo/scripts/apollo_base.sh source /apollo/cyber/setup.bash if [ ${SCENARIO} == "junction" ]; then - python /apollo/modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels_for_junction.py ${SRC_FILE} ${LBL_FILE} + python /apollo/modules/tools/prediction/data_pipelines/data_preprocessing/combine_features_and_labels_for_junction.py ${SRC_FILE} ${LBL_FILE} else - python /apollo/modules/tools/prediction/learning_algorithms/data_preprocessing/combine_features_and_labels.py ${SRC_FILE} ${LBL_FILE} + python /apollo/modules/tools/prediction/data_pipelines/data_preprocessing/combine_features_and_labels.py ${SRC_FILE} ${LBL_FILE} fi diff --git a/modules/tools/prediction/mlp_train/scripts/generate_labels.sh b/modules/tools/prediction/data_pipelines/scripts/generate_labels.sh similarity index 100% rename from modules/tools/prediction/mlp_train/scripts/generate_labels.sh rename to modules/tools/prediction/data_pipelines/scripts/generate_labels.sh diff --git a/modules/tools/prediction/mlp_train/scripts/generate_model_feature.sh b/modules/tools/prediction/data_pipelines/scripts/generate_model_feature.sh similarity index 100% rename from modules/tools/prediction/mlp_train/scripts/generate_model_feature.sh rename to modules/tools/prediction/data_pipelines/scripts/generate_model_feature.sh diff --git a/modules/tools/prediction/learning_algorithms/data_preprocessing/merge_label_dicts_script.sh b/modules/tools/prediction/data_pipelines/scripts/merge_label_dicts_script.sh similarity index 100% rename from modules/tools/prediction/learning_algorithms/data_preprocessing/merge_label_dicts_script.sh rename to modules/tools/prediction/data_pipelines/scripts/merge_label_dicts_script.sh diff --git a/modules/tools/prediction/mlp_train/scripts/records_to_data_for_learning.sh b/modules/tools/prediction/data_pipelines/scripts/records_to_data_for_learning.sh similarity index 100% rename from modules/tools/prediction/mlp_train/scripts/records_to_data_for_learning.sh rename to modules/tools/prediction/data_pipelines/scripts/records_to_data_for_learning.sh diff --git a/modules/tools/prediction/mlp_train/scripts/records_to_dump_feature_proto.sh b/modules/tools/prediction/data_pipelines/scripts/records_to_dump_feature_proto.sh similarity index 100% rename from modules/tools/prediction/mlp_train/scripts/records_to_dump_feature_proto.sh rename to modules/tools/prediction/data_pipelines/scripts/records_to_dump_feature_proto.sh diff --git a/modules/tools/prediction/mlp_train/scripts/records_to_prediction_result.sh b/modules/tools/prediction/data_pipelines/scripts/records_to_prediction_result.sh similarity index 100% rename from modules/tools/prediction/mlp_train/scripts/records_to_prediction_result.sh rename to modules/tools/prediction/data_pipelines/scripts/records_to_prediction_result.sh diff --git a/modules/tools/prediction/mlp_train/feature_extraction.sh b/modules/tools/prediction/mlp_train/feature_extraction.sh deleted file mode 100644 index da410c519b..0000000000 --- a/modules/tools/prediction/mlp_train/feature_extraction.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -############################################################################### -# Copyright 2018 The Apollo Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - -SRC_DIR=$1 -TARGET_DIR=$2 - -set -e - -# TODO(xiaoxq): Remove after rebuilding docker image. -sudo pip install h5py - -source /apollo/scripts/apollo_base.sh -source /apollo/cyber/setup.bash - -FLAGFILE=/apollo/modules/prediction/conf/prediction.conf -echo "--prediction_offline_mode" >> ${FLAGFILE} -echo "--prediction_offline_bags=${SRC_DIR}" >> ${FLAGFILE} -echo "--prediction_data_dir=${TARGET_DIR}" >> ${FLAGFILE} -echo "--junction_distance_threshold=30.0" >> ${FLAGFILE} -echo "--noenable_prioritize_obstacles" >> ${FLAGFILE} - -sudo mkdir -p ${TARGET_DIR} -# sudo chown apollo:apollo ${TARGET_DIR} -cyber_launch start /apollo/modules/prediction/launch/prediction.launch - -for b in $(find ${TARGET_DIR} -iname "*.bin"); do - if [ -e "$b.junction.label" ]; then - echo "skip existing $b.junction.label" - else - echo "Generating junction labels for $b" - python modules/tools/prediction/mlp_train/generate_junction_labels.py \ - $b $b.junction.label - fi - - if [ -e "$b.junction.label.junction.h5" ]; then - echo "skip existing $b.junction.label.junction.h5" - else - echo "Generating junction h5 for $b" - python modules/tools/prediction/mlp_train/generate_junction_h5.py \ - $b.junction.label $b.junction.label.junction.h5 - fi -done - -for b in $(find ${TARGET_DIR} -iname "*.bin"); do - if [ -e "$b.cruise.label" ]; then - echo "skip existing $b.cruise.label" - else - echo "Generating cruise labels for $b" - python modules/tools/prediction/mlp_train/generate_cruise_labels.py \ - $b $b.cruise.label - fi - - if [ -e "$b.cruise.label.cruise.h5" ]; then - echo "skip existing $b.cruise.label.cruise.h5" - else - echo "Generating cruise h5 for $b" - python modules/tools/prediction/mlp_train/generate_cruise_h5.py \ - $b.cruise.label $b.cruise.label.cruise.h5 - fi -done diff --git a/modules/tools/prediction/mlp_train/generate_cruise_h5.py b/modules/tools/prediction/mlp_train/generate_cruise_h5.py deleted file mode 100644 index a77b468714..0000000000 --- a/modules/tools/prediction/mlp_train/generate_cruise_h5.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python - -############################################################################### -# Copyright 2018 The Apollo Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - -import os -import sys -import glob -import argparse -import logging -import numpy as np -import h5py - -from common.configure import parameters -from common.feature_io import load_protobuf -from common.feature_io import load_label_feature -from common.feature_io import save_protobuf -from common.feature_io import build_trajectory -from common.trajectory import TrajectoryToSample - -mlp_feature_size = parameters['cruise_mlp']['dim_input'] -num_output = parameters['cruise_mlp']['dim_output'] - - -def extract_mlp_features(filename): - features = load_label_feature(filename) - - mlp_features = None - for feature in features: - if not feature.HasField('lane') or \ - not feature.lane.HasField('lane_graph'): - continue - lane_seq_sz = len(feature.lane.lane_graph.lane_sequence) - for i in range(lane_seq_sz): - lane_seq = feature.lane.lane_graph.lane_sequence[i] - if len(lane_seq.features.mlp_features) != mlp_feature_size: - continue - mlp_feature = [] - for j in range(mlp_feature_size): - mlp_feature.append(lane_seq.features.mlp_features[j]) - mlp_feature.append(lane_seq.label) - mlp_feature.append(lane_seq.time_to_lane_edge) - mlp_feature.append(lane_seq.time_to_lane_center) - mlp_feature_np = np.array(mlp_feature) - if mlp_features is None: - mlp_features = mlp_feature_np.reshape( - 1, mlp_feature_size+num_output) - else: - mlp_features = np.concatenate( - (mlp_features, mlp_feature_np.reshape( - 1, mlp_feature_size+num_output)), axis=0) - - if (mlp_features is None) or (np.size(mlp_features) == 0): - return None - - return mlp_features - - -def generate_h5_file(filename, output_file): - features = extract_mlp_features(filename) - if (features is None) or (np.size(features) == 0): - print("Failed to extract mlp features from {}".format(filename)) - return - h5_file = h5py.File(output_file, 'w') - h5_file.create_dataset('data', data=features) - h5_file.close() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Generate H5 files') - parser.add_argument('input', type=str, help='input file') - parser.add_argument('output', type=str, help='output file') - args = parser.parse_args() - print("Creating H5: {} -> {}".format(args.input, args.output)) - if os.path.isfile(args.input): - generate_h5_file(args.input, args.output) - else: - print("{} is not a valid file.".format(args.input)) diff --git a/modules/tools/prediction/mlp_train/generate_junction_h5.py b/modules/tools/prediction/mlp_train/generate_junction_h5.py deleted file mode 100644 index 7841f73f93..0000000000 --- a/modules/tools/prediction/mlp_train/generate_junction_h5.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python - -############################################################################### -# Copyright 2018 The Apollo Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - -import os -import sys -import glob -import argparse -import logging -import numpy as np -import h5py - -from common.configure import parameters -from common.feature_io import load_protobuf -from common.feature_io import load_label_feature -from common.feature_io import save_protobuf -from common.feature_io import build_trajectory -from common.trajectory import TrajectoryToSample - -mlp_feature_size = parameters['junction_mlp']['dim_input'] -mlp_label_size = parameters['junction_mlp']['dim_output'] - - -def extract_mlp_features(filename): - features = load_label_feature(filename) - - mlp_features = None - for fea in features: - if not fea.HasField('junction_feature'): - continue - if len(fea.junction_feature.junction_mlp_feature) != mlp_feature_size or \ - len(fea.junction_feature.junction_mlp_label) != mlp_label_size: - continue - mlp_feature = [] - for i in range(mlp_feature_size): - mlp_feature.append(fea.junction_feature.junction_mlp_feature[i]) - for i in range(mlp_label_size): - mlp_feature.append(fea.junction_feature.junction_mlp_label[i]) - mlp_feature_np = np.array(mlp_feature) - if mlp_features is None: - mlp_features = mlp_feature_np.reshape( - 1, mlp_feature_size+mlp_label_size) - else: - mlp_features = np.concatenate( - (mlp_features, mlp_feature_np.reshape(1, mlp_feature_size+mlp_label_size)), axis=0) - if (mlp_features is None) or (np.size(mlp_features) == 0): - return - #mlp_features = mlp_features.reshape( - # (np.shape(mlp_features)[0] / (mlp_feature_size + 2), - # (mlp_feature_size + 2))) - return mlp_features - - -def generate_h5_file(filename, output_file): - features = extract_mlp_features(filename) - if (features is None) or (np.size(features) == 0): - print("Failed to extract mlp features from {}".format(filename)) - return - h5_file = h5py.File(output_file, 'w') - h5_file.create_dataset('data', data=features) - h5_file.close() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Generate H5 files') - parser.add_argument('input', type=str, help='input file') - parser.add_argument('output', type=str, help='output file') - args = parser.parse_args() - print("Creating H5: {} -> {}".format(args.input, args.output)) - if os.path.isfile(args.input): - generate_h5_file(args.input, args.output) - else: - print("{} is not a valid file.".format(args.input)) diff --git a/modules/tools/prediction/mlp_train/generate_output.py b/modules/tools/prediction/mlp_train/generate_output.py deleted file mode 100644 index 1b0ac0c2ed..0000000000 --- a/modules/tools/prediction/mlp_train/generate_output.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python - -############################################################################### -# Copyright 2018 The Apollo Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - -import os -import sys -import glob -import argparse -import logging -import numpy as np -import h5py - -from common.configure import parameters -from common.feature_io import load_protobuf -from common.feature_io import load_label_feature -from common.feature_io import save_protobuf -from common.feature_io import build_trajectory -from common.trajectory import TrajectoryToSample - -mlp_feature_size = parameters['mlp']['dim_input'] - - -def extract_mlp_features(filename): - features = load_label_feature(filename) - - mlp_features = None - for feature in features: - if not feature.HasField('lane') or \ - not feature.lane.HasField('lane_graph'): - continue - lane_seq_sz = len(feature.lane.lane_graph.lane_sequence) - for i in range(lane_seq_sz): - lane_seq = feature.lane.lane_graph.lane_sequence[i] - if len(lane_seq.features.mlp_features) != mlp_feature_size: - continue - mlp_feature = [] - for i in range(mlp_feature_size): - mlp_feature.append(lane_seq.features.mlp_features[i]) - mlp_feature.append(lane_seq.label) - mlp_feature_np = np.array(mlp_feature) - if mlp_features is None: - mlp_features = mlp_feature_np - else: - mlp_features = np.concatenate( - (mlp_features, mlp_feature_np), axis=0) - if (mlp_features is None) or (np.size(mlp_features) == 0): - return - mlp_features = mlp_features.reshape( - (np.shape(mlp_features)[0] / (mlp_feature_size + 1), - (mlp_feature_size + 1))) - return mlp_features - - -def generate_output(filename, output_file): - features = extract_mlp_features(filename) - if (features is None) or (np.size(features) == 0): - print("Failed to extract mlp features from {}".format(filename)) - return - features = features.astype(np.float32) - features.tofile(output_file) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Generate .bin files for tensorflow') - parser.add_argument('input', type=str, help='input file') - parser.add_argument('output', type=str, help='output file') - args = parser.parse_args() - print("Creating .bin: {} -> {}".format(args.input, args.output)) - if os.path.isfile(args.input): - generate_output(args.input, args.output) - else: - print("{} is not a valid file.".format(args.input)) diff --git a/modules/tools/prediction/mlp_train/scripts/generate_feature_proto.sh b/modules/tools/prediction/mlp_train/scripts/generate_feature_proto.sh deleted file mode 100644 index b37ed76a74..0000000000 --- a/modules/tools/prediction/mlp_train/scripts/generate_feature_proto.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -############################################################################### -# Copyright 2019 The Apollo Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### -# -# Use after "apollo.sh build": -# sudo bash /apollo/modules/tools/prediction/mlp_train/scripts/generate_feature_proto.sh -# -# The output feature.X.bin will be in Features proto. - -SRC_DIR=$1 -TARGET_DIR=$2 - -set -e - -source /apollo/scripts/apollo_base.sh -source /apollo/cyber/setup.bash - -FLAGFILE=/apollo/modules/prediction/conf/prediction.conf -echo "--prediction_offline_mode" >> ${FLAGFILE} -echo "--prediction_offline_bags=${SRC_DIR}" >> ${FLAGFILE} -echo "--prediction_data_dir=${TARGET_DIR}" >> ${FLAGFILE} -echo "--junction_distance_threshold=30.0" >> ${FLAGFILE} -echo "--noenable_prioritize_obstacles" >> ${FLAGFILE} - -sudo mkdir -p ${TARGET_DIR} -# sudo chown apollo:apollo ${TARGET_DIR} -cyber_launch start /apollo/modules/prediction/launch/prediction.launch -- GitLab