From 762ff80073a6d9c7b49c37925e97f7a0e0b2b746 Mon Sep 17 00:00:00 2001 From: Advait Jain Date: Thu, 22 Jul 2021 14:21:00 -0700 Subject: [PATCH] Groundwork to enable python format checks (via yapf) as part of the CI. (#314) With this change: * we are set up to use yapf as part of the docker container * pigweed patch is updated to use the google Python style (pep8 with indent of 2 spaces) * some documentation updates. The code still needs to be formatted, and then the Python formatting check will be turned on as part of ci. BUG=http://b/194404216 --- CONTRIBUTING.md | 18 ++++---------- ci/Dockerfile.micro | 5 ++++ docs/continuous_integration.md | 14 +++++++++++ .../lite/micro/tools/make/pigweed.patch | 23 +++++++++++++----- .../lite/micro/tools/make/pigweed_download.sh | 24 +++++++++---------- 5 files changed, 51 insertions(+), 33 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee58e8f4..291eabe3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ @@ -344,22 +344,12 @@ that can be expanded and improved as necessary. * [TensorFlow guide](https://www.tensorflow.org/community/contribute/code_style#python_style) for Python development -* [yapf](https://github.com/google/yapf/) should be used for formatting. +* [yapf](https://github.com/google/yapf/) should be used for formatting. For example: ``` - git clone https://github.com/google/yapf.git - PYTHONPATH= python /yapf log_parser.py -i --style='{based_on_style: pep8, indent_width: 2}' + pip3 install yapf + yapf log_parser.py -i --style='{based_on_style: pep8, indent_width: 2}' ``` # Continuous Integration System - * See the [github workflow files](.github/workflows/ci.yml) for details on - exactly what is run as part of the GitHub Actions CI. - * Some [additional documentation](docs/continuous_integration.md) on the TFLM CI. - - * Tests can also be run from within a docker container: - ``` - docker build -t tflm-test -f ci/Dockerfile.micro . - docker run --rm tflm-test - ``` - diff --git a/ci/Dockerfile.micro b/ci/Dockerfile.micro index a9a1d419..7ce73bb0 100644 --- a/ci/Dockerfile.micro +++ b/ci/Dockerfile.micro @@ -27,7 +27,12 @@ RUN ln -s /usr/bin/clang-12 /usr/bin/clang RUN ln -s /usr/bin/clang++-12 /usr/bin/clang++ RUN ln -s /usr/bin/clang-format-12 /usr/bin/clang-format +# Install yapf to check for Python formatting as part of the TFLM continuous +# integration. +RUN pip install yapf + RUN pip install six + # Install Renode test dependencies RUN pip install pyyaml requests psutil robotframework==3.1 diff --git a/docs/continuous_integration.md b/docs/continuous_integration.md index f56717e5..0e1585f2 100644 --- a/docs/continuous_integration.md +++ b/docs/continuous_integration.md @@ -8,6 +8,20 @@ [TensorFlow repo]: https://github.com/tensorflow/tensorflow +# Continuous Integration System + * See the [github workflow files](.github/workflows/ci.yml) for details on + exactly what is run as part of the GitHub Actions CI. + + * Tests can also be run from within a docker container: + ``` + docker build -t tflm-test -f ci/Dockerfile.micro . + ``` + + or use the tflm-ci docker image from [here](https://github.com/users/TFLM-bot/packages/container/package/tflm-ci). + + * You will still need to copy or mount your fork of tflite-micro on to this + docker container prior to running any tests. + # Automated Sync from the Tensorflow Repository While TfLite Micro and TfLite are in separate GitHub repositories, the two diff --git a/tensorflow/lite/micro/tools/make/pigweed.patch b/tensorflow/lite/micro/tools/make/pigweed.patch index 89dba14d..d1e29304 100644 --- a/tensorflow/lite/micro/tools/make/pigweed.patch +++ b/tensorflow/lite/micro/tools/make/pigweed.patch @@ -11,19 +11,30 @@ index 4a370e33..224ad9c6 100644 _LOG = logging.getLogger(__name__) diff --git a/pw_presubmit/py/pw_presubmit/format_code.py b/pw_presubmit/py/pw_presubmit/format_code.py -index 19d09546..dae2e813 100755 +index 19d09546..c1ff6b5a 100755 --- a/pw_presubmit/py/pw_presubmit/format_code.py +++ b/pw_presubmit/py/pw_presubmit/format_code.py -@@ -229,8 +229,7 @@ def print_format_check(errors: Dict[Path, str], +@@ -142,7 +142,7 @@ def fix_go_format(files: Iterable[Path]) -> None: + + + def _yapf(*args, **kwargs) -> subprocess.CompletedProcess: +- return log_run(['python', '-m', 'yapf', '--parallel', *args], ++ return log_run(['python', '-m', 'yapf', '--style', '{based_on_style:pep8,indent_width:2}', '--parallel', *args], + capture_output=True, + **kwargs) + +@@ -229,11 +229,6 @@ def print_format_check(errors: Dict[Path, str], except ValueError: return Path(path).resolve() - message = (f' pw format --fix {path_relative_to_cwd(path)}' - for path in errors) -+ message = (f' tensorflow/lite/{__file__} --fix {path}' for path in errors) - _LOG.warning('To fix formatting, run:\n\n%s\n', '\n'.join(message)) - - +- _LOG.warning('To fix formatting, run:\n\n%s\n', '\n'.join(message)) +- +- + class CodeFormat(NamedTuple): + language: str + extensions: Collection[str] diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py index 794967db..061db7ea 100755 --- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py diff --git a/tensorflow/lite/micro/tools/make/pigweed_download.sh b/tensorflow/lite/micro/tools/make/pigweed_download.sh index 9991ee8b..482d2582 100755 --- a/tensorflow/lite/micro/tools/make/pigweed_download.sh +++ b/tensorflow/lite/micro/tools/make/pigweed_download.sh @@ -31,21 +31,18 @@ set -e +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR=${SCRIPT_DIR}/../../../../.. +cd "${ROOT_DIR}" + +source tensorflow/lite/micro/tools/make/bash_helpers.sh + DOWNLOADS_DIR=${1} if [ ! -d ${DOWNLOADS_DIR} ]; then echo "The top-level downloads directory: ${DOWNLOADS_DIR} does not exist." exit 1 fi -# The BUILD files in the downloaded folder result in an error with: -# bazel build tensorflow/lite/micro/... -# -# Parameters: -# $1 - path to the downloaded flatbuffers code. -function delete_build_files() { - rm -f `find ${1} -name BUILD` -} - DOWNLOADED_PIGWEED_PATH=${DOWNLOADS_DIR}/pigweed if [ -d ${DOWNLOADED_PIGWEED_PATH} ]; then @@ -53,13 +50,14 @@ if [ -d ${DOWNLOADED_PIGWEED_PATH} ]; then else git clone https://pigweed.googlesource.com/pigweed/pigweed ${DOWNLOADED_PIGWEED_PATH} >&2 pushd ${DOWNLOADED_PIGWEED_PATH} > /dev/null + git checkout 47268dff45019863e20438ca3746c6c62df6ef09 >&2 + rm -rf ${DOWNLOADED_PIGWEED_PATH}/.git + rm -f `find . -name BUILD` - # Patch for TFLM specific changes that are not currently upstreamed. - git apply ../../pigweed.patch - popd > /dev/null + apply_patch_to_folder ./ ../../pigweed.patch - delete_build_files ${DOWNLOADED_PIGWEED_PATH} + popd > /dev/null fi echo "SUCCESS" -- GitLab