PythonUtil.h 9.6 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#pragma once
Y
Yu Yang 已提交
16
// clang-format off
X
Xin Pan 已提交
17
#include "paddle/legacy/utils/Util.h"
Z
zhangjinchao01 已提交
18 19 20 21

#ifndef PADDLE_NO_PYTHON
// must include the following two blocks, otherwise,
// gcc compiler may produce warning
L
liaogang 已提交
22 23 24 25 26 27
#ifdef __APPLE__
#define _POSIX_SOURCE
#define _POSIX_C_SOURCE 200809L
#define _XOPEN_SOURCE 700
#endif

Z
zhangjinchao01 已提交
28 29 30 31 32 33 34 35 36 37 38 39 40 41
#ifdef _POSIX_C_SOURCE
#define __TEMP_POSIX_C_SOURCE _POSIX_C_SOURCE
#undef _POSIX_C_SOURCE
#endif
#ifdef _XOPEN_SOURCE
#define __TEMP_XOPEN_SOURCE _XOPEN_SOURCE
#undef _XOPEN_SOURCE
#endif
#include <Python.h>
#include <frameobject.h>
#endif

#include <stdarg.h>
#include <map>
Y
Yu Yang 已提交
42
#include <mutex>
Y
Yu Yang 已提交
43
// clang-format on
Z
zhangjinchao01 已提交
44 45 46 47 48 49 50 51 52 53 54 55 56 57

namespace paddle {

std::string callPythonFunc(const std::string& moduleName,
                           const std::string& funcName,
                           const std::vector<std::string>& args);

#ifndef PADDLE_NO_PYTHON

/**
 * Global lock guard of python C-api invokes.
 * NOTE: the lock of this guard is reentrant or recursive.
 */
class PyGuard {
W
Wu Yi 已提交
58
 public:
Z
zhangjinchao01 已提交
59 60 61 62
  PyGuard();
  PyGuard(const PyGuard& other) = delete;
  PyGuard& operator=(const PyGuard& other) = delete;

W
Wu Yi 已提交
63
 private:
Z
zhangjinchao01 已提交
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
  std::lock_guard<std::recursive_mutex> guard_;
};

struct PyObjectDeleter {
  void operator()(PyObject* obj) {
    if (obj) {
      Py_DECREF(obj);
    }
  }
};

typedef std::unique_ptr<PyObject, PyObjectDeleter> PyObjectPtr;

PyObjectPtr callPythonFuncRetPyObj(const std::string& moduleName,
                                   const std::string& funcName,
                                   const std::vector<std::string>& args);

PyObjectPtr createPythonClass(const std::string& moduleName,
                              const std::string& className,
                              const std::vector<std::string>& args,
                              const std::map<std::string, std::string>& kwargs);

86
#define CHECK_PY(x) CHECK((x) != nullptr) << ::paddle::py::getPyCallStack()
Z
zhangjinchao01 已提交
87 88

namespace py {
89 90
PyObjectPtr import(const std::string& moduleName);

M
minqiyang 已提交
91
#if PY_MAJOR_VERSION >= 3
M
minqiyang 已提交
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
/**
 * Cast a PyLong to int type T.
 * @tparam T return type.
 * @param [in] obj PyLong object.
 * @param [out] ok status for casting. False if error occured. nullptr if user
 *                 don't care is ok or not.
 * @return The value of python object, or 0 if not ok.
 */
template <typename T>
T castInt(PyObject* obj, bool* ok = nullptr) {
  // Refer to https://www.python.org/dev/peps/pep-0237/, the int and long object
  // were unified to long since python3
  if (PyLong_Check(obj)) {
    if (ok) *ok = true;
    return (T)PyLong_AsUnsignedLong(obj);
  } else {
    if (ok) *ok = false;
    return (T)0;
  }
}

// Convert PyAPI from 2.x to 3.x
#define PyString_FromString PyUnicode_FromString
#define PyString_AsString PyUnicode_AsUTF8

#else
Z
zhangjinchao01 已提交
118 119 120 121 122 123 124 125 126 127
/**
 * Cast a PyLong or PyInt to int type T.
 * @tparam T return type.
 * @param [in] obj PyLong or PyInt object.
 * @param [out] ok status for casting. False if error occured. nullptr if user
 *                 don't care is ok or not.
 * @return The value of python object, or 0 if not ok.
 */
template <typename T>
T castInt(PyObject* obj, bool* ok = nullptr) {
M
minqiyang 已提交
128
  if (PyLong_Check(obj)) {
Z
zhangjinchao01 已提交
129
    if (ok) *ok = true;
130
    return (T)PyLong_AsUnsignedLong(obj);
M
minqiyang 已提交
131
  } else if (PyInt_Check(obj)) {
Z
zhangjinchao01 已提交
132
    if (ok) *ok = true;
M
minqiyang 已提交
133
    return (T)PyInt_AsLong(obj);
Z
zhangjinchao01 已提交
134 135
  } else {
    if (ok) *ok = false;
136
    return (T)0;
Z
zhangjinchao01 已提交
137 138
  }
}
M
minqiyang 已提交
139
#endif // PY_MAJOR_VERSION >= 3
Z
zhangjinchao01 已提交
140 141 142 143 144 145

/**
 * Invoke repr of python object.
 *
 * Just like toString method in java.
 */
146
char* repr(PyObject* obj);
Z
zhangjinchao01 已提交
147 148 149 150

/**
 * Invoke repr of python object.
 */
151
inline char* repr(const PyObjectPtr& obj) { return repr(obj.get()); }
Z
zhangjinchao01 已提交
152 153 154 155 156 157 158 159 160 161 162 163

/**
 * Get Python Error Stack String.
 */
std::string getPyCallStack();

/**
 * Object Helper for PyObjectPtr.
 *
 * Implements getAttr method for object.
 */
class ObjectHelper {
W
Wu Yi 已提交
164
 public:
165
  explicit ObjectHelper(const PyObjectPtr& obj) : obj_(obj) {}
Z
zhangjinchao01 已提交
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204

  /**
   * get attribute
   */
  inline PyObject* getAttr(const std::string& field) const {
    auto obj = PyObject_GetAttrString(obj_.get(), field.c_str());
    CHECK_PY(obj) << "Cannot get attribute on python object " << obj_.get();
    return obj;
  }

  /**
   * Get Int attribute
   * @param [in] field  attribute name.
   * @param [out] ok true if this attribute is int.
   * @tparam T int type.
   * @return int value.
   */
  template <typename T>
  T getIntAttr(const std::string& field, bool* ok = nullptr) const {
    PyObjectPtr tmp(getAttr(field));
    return castInt<T>(tmp.get(), ok);
  }

  /**
   * Get int attribute. Log(Fatal) when not ok
   * @param field attribute name.
   * @return int value.
   */
  template <typename T>
  T getIntAttrWithError(const std::string& field) const {
    bool ok;
    T tmp = getIntAttr<T>(field, &ok);
    CHECK(ok) << "Cannot get integer attribute on object " << obj_.get();
    return tmp;
  }

  /**
   * Get bool attribute.
   * @param field
205 206 207 208 209 210 211 212
   * @param [out] isBoolType return true if attribute is bool type. If the
   *                         attribute is not bool type, then an implicit
   *                         conversion will happens, and will return the
   *                         conversion result.
   *
   *                         Such as, if the attribute is 1, then the return
   *                         value of function will be true, but the isBoolType
   *                         will return false.
Z
zhangjinchao01 已提交
213 214
   * @return
   */
215
  bool getBoolAttr(const std::string& field, bool* isBoolType = nullptr) const {
Z
zhangjinchao01 已提交
216
    PyObjectPtr tmp(getAttr(field));
217 218 219
    if (isBoolType) {
      *isBoolType = PyBool_Check(tmp.get());
    }
Z
zhangjinchao01 已提交
220 221 222
    return PyObject_IsTrue(tmp.get());
  }

W
Wu Yi 已提交
223
 private:
Z
zhangjinchao01 已提交
224 225 226 227 228 229 230 231 232
  const PyObjectPtr& obj_;
};

/**
 * Python Sequence Helper
 *
 * The python sequence means list or tuple.
 */
class SequenceHelper {
W
Wu Yi 已提交
233
 public:
Z
zhangjinchao01 已提交
234 235 236 237
  explicit SequenceHelper(const PyObjectPtr& seq) : seq_(seq.get()) {
    CHECK(PySequence_Check(seq_));
  }

238
  explicit SequenceHelper(PyObject* seq) : seq_(seq) {
Z
zhangjinchao01 已提交
239 240 241
    CHECK(PySequence_Check(seq_));
  }

242
  inline size_t size() const { return (size_t)PySequence_Size(seq_); }
Z
zhangjinchao01 已提交
243

244
  inline PyObject* operator[](size_t i) const {
Z
zhangjinchao01 已提交
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
    return PySequence_Fast_GET_ITEM(seq_, i);
  }

  inline double getDouble(size_t i) const {
    auto* ptr = (*this)[i];
    return PyFloat_AsDouble(ptr);
  }

  /**
   * Set a sequence item o[i] = obj;
   * @param i index
   * @param obj setted item.
   * @param steal if steal = true, sequence will move object in iteself,
   *              just like std::move. Otherwise, it will increase reference
   *              count. Default is false.
   */
  inline void set(size_t i, const PyObjectPtr& obj, bool steal = false) {
    this->set(i, obj.get(), steal);
  }

  /**
   * Set a sequence item o[i] = obj;
   */
  inline void set(size_t i, PyObject* obj, bool steal = false) {
    if (!steal) {
      Py_XINCREF(obj);
    }
    if (PyTuple_Check(seq_)) {
      CHECK_NE(PyTuple_SetItem(seq_, i, obj), -1) << getPyCallStack();
    } else {
      CHECK_NE(PySequence_SetItem(seq_, i, obj), -1) << getPyCallStack();
    }
  }

W
Wu Yi 已提交
279
 private:
Z
zhangjinchao01 已提交
280 281 282 283
  PyObject* seq_;
};

class DictHelper {
W
Wu Yi 已提交
284
 public:
285
  explicit DictHelper(PyObject* d) : dict_(d) {}
Z
zhangjinchao01 已提交
286

287
  explicit DictHelper(const PyObjectPtr& d) : dict_(d.get()) {}
Z
zhangjinchao01 已提交
288 289 290 291 292 293 294 295 296

  void set(const std::string& key, PyObject* item) {
    PyDict_SetItemString(dict_, key.c_str(), item);
  }

  void setBool(const std::string& key, bool b) {
    this->set(key, PyBool_FromLong(b));
  }

297 298
  void setStringList(const std::string& key,
                     const std::vector<std::string>& items) {
299 300
    auto* list = PyList_New(items.size());
    for (size_t i = 0; i < items.size(); ++i) {
301 302 303 304 305
      PyList_SetItem(list, i, PyString_FromString(items[i].c_str()));
    }
    this->set(key, list);
  }

W
Wu Yi 已提交
306
 private:
307
  inline void checkDict() { CHECK(PyDict_Check(this->dict_)); }
Z
zhangjinchao01 已提交
308 309 310 311 312 313 314 315 316 317 318 319

  PyObject* dict_;
};

inline static bool isCallable(const PyObjectPtr& obj) {
  return PyCallable_Check(obj.get());
}

/**
 * Wrap a callable object.
 */
class CallableHelper {
W
Wu Yi 已提交
320
 public:
321
  explicit CallableHelper(const PyObjectPtr& obj) : obj_(obj) {
Z
zhangjinchao01 已提交
322 323 324 325 326 327 328 329 330
    CHECK(py::isCallable(obj_));
  }

  ~CallableHelper() {}

  /**
   * reset args, and create new tuple.
   * @param sz args size.
   */
331
  void setArgsSize(size_t sz) { args.reset(PyTuple_New(sz)); }
Z
zhangjinchao01 已提交
332 333 334 335

  /**
   * Get args sequence. User can set/get by SequenceHelper.
   */
336
  SequenceHelper getArgs() { return SequenceHelper(args); }
Z
zhangjinchao01 已提交
337 338 339 340

  /**
   * Call python method, return an object.
   */
341
  PyObject* operator()() {
Z
zhangjinchao01 已提交
342 343 344 345
    PyGuard guard;
    return PyObject_Call(obj_.get(), args.get(), kwargs.get());
  }

W
Wu Yi 已提交
346
 private:
Z
zhangjinchao01 已提交
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
  const PyObjectPtr& obj_;
  PyObjectPtr args;
  PyObjectPtr kwargs;
};

inline static PyObject* iterNext(const PyObjectPtr& context, bool* atEnd) {
  PyGuard g;
  PyObject* data = PyIter_Next(context.get());
  if (data == nullptr) {
    if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
      PyErr_Clear();
      *atEnd = true;
      return nullptr;
    } else if (PyErr_Occurred()) {
      CHECK_PY(data) << "Calling iterator next error";
      return nullptr;
    } else {
      *atEnd = false;
      return data;  // just return none in iterator.
    }
  } else {
    *atEnd = false;
    return data;
  }
}
}  // namespace py

#endif

/**
 * Initialize python.
 */
void initPython(int argc, char** argv);

}  // namespace paddle