val_and_test.py

import os
import sys
import argparse
import time
import traceback
import subprocess
import re

import utils
import infer
import config
from utils import logger


def load_existing_results(eval_result_file):
    evals = {}
    with utils.open_file(eval_result_file) as f:
        for line in f:
            line = line.strip()
            if not line: continue
            pos = line.find(" ")
            pass_id, ret = int(line[len("Pass="):pos]), line[pos + 1:]
            evals[pass_id] = ret
    return evals


__PATTERN_CHUNK_F1 = re.compile("chunk_f1=(\d+(\.\d+)?)")


def find_best_pass(evals):
    results = []
    for pass_id, eval_ret in evals.iteritems():
        chunk_f1 = float(__PATTERN_CHUNK_F1.search(eval_ret).group(1))
        results.append((pass_id, chunk_f1))

    results.sort(key=lambda item: (-item[1], item[0]))
    return results[0][0]


def eval_one_pass(infer_obj, conf, model_path, data_path, eval_script):
    if not os.path.exists("tmp"): os.makedirs("tmp")
    # model file is not ready
    if not os.path.exists(model_path): return False

    output_path = os.path.join("tmp", "%s_%s.txt.gz" % (
        os.path.basename(model_path), os.path.basename(data_path)))
    with utils.open_file(output_path, "w") as output:
        try:
            infer_obj.infer(model_path, data_path, output)
        except Exception as ex:
            traceback.print_exc()
            return None

    cmd = [
        "python", eval_script, output_path, data_path, "--fuzzy", "--schema",
        conf.label_schema
    ]
    logger.info("cmd: %s" % " ".join(cmd))
    eval_ret = subprocess.check_output(cmd)
    if "chunk_f1" not in eval_ret:
        raise ValueError("Unknown error in cmd \"%s\"" % " ".join(cmd))

    return eval_ret


def run_eval(infer_obj,
             conf,
             model_dir,
             input_path,
             eval_script,
             log_file,
             start_pass_id,
             end_pass_id,
             force_rerun=False):
    if not force_rerun and os.path.exists(log_file):
        evals = load_existing_results(log_file)
    else:
        evals = {}
    with utils.open_file(log_file, "w") as log:
        for i in xrange(start_pass_id, end_pass_id + 1):
            if i in evals:
                eval_ret = evals[i]
            else:
                pass_id = "%05d" % i
                model_path = os.path.join(model_dir,
                                          "params_pass_%s.tar.gz" % pass_id)
                logger.info("Waiting for model %s ..." % model_path)
                while True:
                    eval_ret = eval_one_pass(infer_obj, conf, model_path,
                                             input_path, eval_script)
                    if eval_ret:
                        evals[i] = eval_ret
                        break

                    # wait for one minute and rerun
                    time.sleep(60)
            print >> log, "Pass=%d %s" % (i, eval_ret.rstrip())
            log.flush()
    return evals


def parse_cmd():
    parser = argparse.ArgumentParser()
    parser.add_argument("model_dir")
    parser.add_argument("data_type", choices=["ann", "ir"], default="ann")
    parser.add_argument(
        "--val_eval_output", help="validation set evaluation result file")
    parser.add_argument(
        "--tst_eval_output", help="test set evaluation result file")
    parser.add_argument("--start_pass_id", type=int, default=0)
    parser.add_argument(
        "--end_pass_id", type=int, default=24, help="this pass is included")
    parser.add_argument("--force_rerun", action="store_true")
    return parser.parse_args()


__eval_scripts = {
    "ann": "data/evaluation/evaluate-tagging-result.py",
    "ir": "data/evaluation/evaluate-voting-result.py",
}

__val_data = {
    "ann": "./data/data/validation.ann.json.gz",
    "ir": "./data/data/validation.ir.json.gz",
}

__tst_data = {
    "ann": "./data/data/test.ann.json.gz",
    "ir": "./data/data/test.ir.json.gz",
}


def main(args):
    conf = config.InferConfig()
    conf.vocab = utils.load_dict(conf.word_dict_path)
    logger.info("length of word dictionary is : %d." % len(conf.vocab))

    if args.val_eval_output:
        val_eval_output = args.val_eval_output
    else:
        val_eval_output = "eval.val.%s.txt" % args.data_type

    if args.tst_eval_output:
        tst_eval_output = args.tst_eval_output
    else:
        tst_eval_output = "eval.tst.%s.txt" % args.data_type

    eval_script = __eval_scripts[args.data_type]
    val_data_file = __val_data[args.data_type]
    tst_data_file = __tst_data[args.data_type]

    infer_obj = infer.Infer(conf)
    val_evals = run_eval(
        infer_obj,
        conf,
        args.model_dir,
        val_data_file,
        eval_script,
        val_eval_output,
        args.start_pass_id,
        args.end_pass_id,
        force_rerun=args.force_rerun)

    best_pass_id = find_best_pass(val_evals)

    tst_evals = run_eval(
        infer_obj,
        conf,
        args.model_dir,
        tst_data_file,
        eval_script,
        tst_eval_output,
        start_pass_id=best_pass_id,
        end_pass_id=best_pass_id,
        force_rerun=args.force_rerun)

    logger.info("Best Pass=%d" % best_pass_id)
    logger.info("Validation: %s" % val_evals[best_pass_id])
    logger.info("Test      : %s" % tst_evals[best_pass_id])


if __name__ == "__main__":
    main(parse_cmd())