PythonUtil.cpp.in 6.9 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "PythonUtil.h"
#include <signal.h>
Y
Yu Yang 已提交
17
#include <sstream>
Z
zhangjinchao01 已提交
18 19 20 21 22

namespace paddle {

#ifdef PADDLE_NO_PYTHON

23 24
DEFINE_string(python_path, "", "python path");
DEFINE_string(python_bin, "python2.7", "python bin");
Z
zhangjinchao01 已提交
25 26 27 28 29 30 31 32 33 34

constexpr int kExecuteCMDBufLength = 204800;

int executeCMD(const char* cmd, char* result) {
  char bufPs[kExecuteCMDBufLength];
  char ps[kExecuteCMDBufLength] = {0};
  FILE* ptr;
  strncpy(ps, cmd, kExecuteCMDBufLength);
  if ((ptr = popen(ps, "r")) != NULL) {
    size_t count = fread(bufPs, 1, kExecuteCMDBufLength, ptr);
35 36
    memcpy(result,
           bufPs,
Z
zhangjinchao01 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
           count - 1);  // why count-1: remove the '\n' at the end
    result[count] = 0;
    pclose(ptr);
    ptr = NULL;
    return count - 1;
  } else {
    LOG(FATAL) << "popen failed";
    return -1;
  }
}

std::string callPythonFunc(const std::string& moduleName,
                           const std::string& funcName,
                           const std::vector<std::string>& args) {
  std::string pythonLibPath = "";
  std::string pythonBinPath = "";
  if (!FLAGS_python_path.empty()) {
    pythonLibPath = FLAGS_python_path + "/lib:";
    pythonBinPath = FLAGS_python_path + "/bin/";
  }
  std::string s = "LD_LIBRARY_PATH=" + pythonLibPath + "$LD_LIBRARY_PATH " +
                  pythonBinPath + std::string(FLAGS_python_bin) +
                  " -c 'import " + moduleName + "\n" + "print " + moduleName +
                  "." + funcName + "(";
  for (auto& arg : args) {
    s = s + "\"" + arg + "\", ";
  }
  s += ")'";
  char result[kExecuteCMDBufLength] = {0};
  LOG(INFO) << " cmd string: " << s;
  int length = executeCMD(s.c_str(), result);
  CHECK_NE(-1, length);
  return std::string(result, length);
}

#else

static std::recursive_mutex g_pyMutex;

PyGuard::PyGuard() : guard_(g_pyMutex) {}

78 79
static void printPyErrorStack(std::ostream& os,
                              bool withEndl = false,
80
                              bool withPyPath = true) {
81
  PyObject *ptype, *pvalue, *ptraceback;
Z
zhangjinchao01 已提交
82 83 84
  PyErr_Fetch(&ptype, &pvalue, &ptraceback);
  PyErr_NormalizeException(&ptype, &pvalue, &ptraceback);
  PyErr_Clear();
85 86 87 88 89 90
  if (withPyPath) {
    os << "Current PYTHONPATH: " << py::repr(PySys_GetObject(strdup("path")));
    if (withEndl) {
      os << std::endl;
    }
  }
Z
zhangjinchao01 已提交
91 92
  PyTracebackObject* obj = (PyTracebackObject*)ptraceback;

93 94
  os << "Python Error: " << PyString_AsString(PyObject_Str(ptype)) << " : "
     << (pvalue == NULL ? "" : PyString_AsString(PyObject_Str(pvalue)));
Z
zhangjinchao01 已提交
95 96 97 98 99 100 101 102 103
  if (withEndl) {
    os << std::endl;
  }
  os << "Python Callstack: ";
  if (withEndl) {
    os << std::endl;
  }
  while (obj != NULL) {
    int line = obj->tb_lineno;
104 105
    const char* filename =
        PyString_AsString(obj->tb_frame->f_code->co_filename);
Z
zhangjinchao01 已提交
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
    os << "            " << filename << " : " << line;
    if (withEndl) {
      os << std::endl;
    }
    obj = obj->tb_next;
  }

  Py_XDECREF(ptype);
  Py_XDECREF(pvalue);
  Py_XDECREF(ptraceback);
}
PyObjectPtr callPythonFuncRetPyObj(const std::string& moduleName,
                                   const std::string& funcName,
                                   const std::vector<std::string>& args) {
  PyGuard guard;
121
  PyObjectPtr pyModule = py::import(moduleName);
Z
zhangjinchao01 已提交
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
  PyObjectPtr pyFunc(PyObject_GetAttrString(pyModule.get(), funcName.c_str()));
  CHECK_PY(pyFunc) << "GetAttrString failed.";
  PyObjectPtr pyArgs(PyTuple_New(args.size()));
  for (size_t i = 0; i < args.size(); ++i) {
    PyObjectPtr pyArg(PyString_FromString(args[i].c_str()));
    CHECK_PY(pyArg) << "Import pyArg failed.";
    PyTuple_SetItem(pyArgs.get(), i, pyArg.release());  //  Maybe a problem
  }
  PyObjectPtr ret(PyObject_CallObject(pyFunc.get(), pyArgs.get()));
  CHECK_PY(ret) << "Call Object failed.";
  return ret;
}

std::string callPythonFunc(const std::string& moduleName,
                           const std::string& funcName,
                           const std::vector<std::string>& args) {
  PyObjectPtr obj = callPythonFuncRetPyObj(moduleName, funcName, args);
  return std::string(PyString_AsString(obj.get()), PyString_Size(obj.get()));
}

PyObjectPtr createPythonClass(
143 144
    const std::string& moduleName,
    const std::string& className,
Z
zhangjinchao01 已提交
145 146 147
    const std::vector<std::string>& args,
    const std::map<std::string, std::string>& kwargs) {
  PyGuard guard;
148
  PyObjectPtr pyModule = py::import(moduleName);
Z
zhangjinchao01 已提交
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
  LOG(INFO) << "createPythonClass moduleName.c_str:" << moduleName.c_str();
  CHECK_PY(pyModule) << "Import module " << moduleName << " failed.";
  PyObjectPtr pyDict(PyModule_GetDict(pyModule.get()));
  CHECK_PY(pyDict) << "Get Dict failed.";
  PyObjectPtr pyClass(PyDict_GetItemString(pyDict.get(), className.c_str()));
  LOG(INFO) << "createPythonClass className.c_str():" << className.c_str();
  CHECK_PY(pyClass) << "Import class " << className << " failed.";
  PyObjectPtr argsObjectList(PyTuple_New(args.size()));
  for (size_t i = 0; i < args.size(); ++i) {
    PyObjectPtr pyArg(Py_BuildValue("s#", args[i].c_str(), args[i].length()));
    PyTuple_SetItem(argsObjectList.get(), i, pyArg.release());
  }

  PyObjectPtr kwargsObjectList(PyDict_New());
  for (auto& x : kwargs) {
    PyObjectPtr pyArg(Py_BuildValue("s#", x.second.c_str(), x.second.length()));
165 166
    PyDict_SetItemString(
        kwargsObjectList.get(), x.first.c_str(), pyArg.release());
Z
zhangjinchao01 已提交
167 168
  }

169 170
  PyObjectPtr pyInstance(PyInstance_New(
      pyClass.get(), argsObjectList.release(), kwargsObjectList.release()));
Z
zhangjinchao01 已提交
171 172 173 174 175
  CHECK_PY(pyInstance) << "Create class " << className << " failed.";
  return pyInstance;
}

namespace py {
176
char* repr(PyObject* obj) { return PyString_AsString(PyObject_Repr(obj)); }
Z
zhangjinchao01 已提交
177 178 179 180 181 182

std::string getPyCallStack() {
  std::ostringstream os;
  printPyErrorStack(os, true);
  return os.str();
}
183

184
PyObjectPtr import(const std::string& moduleName) {
185 186 187 188 189
  auto module = PyImport_ImportModule(moduleName.c_str());
  CHECK_PY(module) << "Import " << moduleName << "Error";
  return PyObjectPtr(module);
}

Z
zhangjinchao01 已提交
190 191 192
}  // namespace py

#endif
193 194 195
extern "C" {
extern const char enable_virtualenv_py[];
}
Z
zhangjinchao01 已提交
196 197
void initPython(int argc, char** argv) {
#ifndef PADDLE_NO_PYTHON
198 199 200 201
  char pyHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
  if (strlen(pyHome)) {
    Py_SetPythonHome(pyHome);
  }
Z
zhangjinchao01 已提交
202 203 204 205 206
  Py_SetProgramName(argv[0]);
  Py_Initialize();
  PySys_SetArgv(argc, argv);
  // python blocks SIGINT. Need to enable it.
  signal(SIGINT, SIG_DFL);
207 208 209

  // Manually activate virtualenv when user is using virtualenv
  PyRun_SimpleString(enable_virtualenv_py);
Z
zhangjinchao01 已提交
210 211 212 213
#endif
}

}  // namespace paddle