Arduino python (#257)

* Add files from TfLite

Added batch_matmul.h and tensor_utils_common.h that are needed for the BATCH_MATMUL operator.

* Arduino library creation for CI build.
Issue# 256

* fix bug with include file path flattening

* fix examples path generation

Some files in examples source and headers list are not in the examples directory.  They can be tensorflow files or third-party files.  The code is corrected so that these files end up inside the generated directory tree.  Previously, these files were being placed outside the generated directory tree and were thus invisible to follow-on targeted generation scripts.

* remove/revert files that go into the arduino-examples repo

* Format with:

```
tensorflow/lite/micro/tools/ci_build/test_code_style.sh --fix_formatting
```

* request changes to PR

* Removed symbolic link for micro_speech example from Makefile.
Added additional include flags to Makefile.
Adjusted header file transformations during base tree creation.
Co-authored-by: NAdvait Jain <advaitjain@users.noreply.github.com>
Co-authored-by: NAdvait Jain <advaitjain@google.com>

tensorflow/lite/micro/examples/magic_wand/Makefile.inc

@@ -82,3 +82,9 @@ $(magic_wand_TEST_SRCS),$(magic_wand_TEST_HDRS)))
 # Builds a standalone binary
 $(eval $(call microlite_test,magic_wand,\
 $(magic_wand_SRCS),$(magic_wand_HDRS)))
+
+list_magic_wand_example_sources:
+	@echo $(magic_wand_SRCS)
+
+list_magic_wand_example_headers:
+	@echo $(magic_wand_HDRS)

tensorflow/lite/micro/examples/micro_speech/Makefile.inc

@@ -284,3 +284,9 @@ $(MICRO_SPEECH_SRCS),$(MICRO_SPEECH_HDRS)))
 # Builds a standalone speech command recognizer binary using fake audio input.
 $(eval $(call microlite_test,micro_speech_mock,\
 $(MICRO_SPEECH_MOCK_SRCS),$(MICRO_SPEECH_MOCK_HDRS)))
+
+list_micro_speech_example_sources:
+	@echo $(MICRO_SPEECH_SRCS)
+
+list_micro_speech_example_headers:
+	@echo $(MICRO_SPEECH_HDRS)

tensorflow/lite/micro/examples/person_detection/Makefile.inc

@@ -79,3 +79,9 @@ $(DETECTION_RESPONDER_TEST_SRCS),$(DETECTION_RESPONDER_TEST_HDRS)))
 # Builds a standalone object recognition binary.
 $(eval $(call microlite_test,person_detection_int8,\
 $(person_detection_SRCS),$(person_detection_HDRS)))
+
+list_person_detection_example_sources:
+	@echo $(person_detection_SRCS)
+
+list_person_detection_example_headers:
+	@echo $(person_detection_HDRS)

tensorflow/lite/micro/tools/ci_build/test_project_generation.sh

@@ -27,7 +27,10 @@ TEST_OUTPUT_DIR=$(mktemp -d)
 readable_run \
   python3 tensorflow/lite/micro/tools/project_generation/create_tflm_tree.py \
   ${TEST_OUTPUT_DIR} \
-  -e hello_world
+  -e hello_world \
+  -e magic_wand \
+  -e micro_speech \
+  -e person_detection
 
 # Confirm that print_src_files and print_dest_files output valid paths (and
 # nothing else).

tensorflow/lite/micro/tools/project_generation/Makefile

@@ -67,4 +67,26 @@ hello_world: libtflm
 	@mkdir -p $(BINDIR)
 	$(CXX) $(CXXFLAGS) $(wildcard examples/hello_world/*.cc) $(INCLUDES) $(LIB) -o $(BINDIR)/$@
 
-examples: hello_world
+magic_wand: libtflm
+	@mkdir -p $(BINDIR)
+	$(CXX) $(CXXFLAGS) $(wildcard examples/magic_wand/*.cc) $(INCLUDES) $(LIB) -o $(BINDIR)/$@
+
+MICRO_SPEECH_SRCS := $(wildcard examples/micro_speech/*.cc)
+MICRO_SPEECH_SRCS += $(wildcard examples/micro_speech/*/*.cc)
+MICRO_SPEECH_THIRD_PARTY_SRCS := $(wildcard third_party/kissfft/*.c)
+MICRO_SPEECH_THIRD_PARTY_SRCS += $(wildcard third_party/kissfft/*/*.c)
+MICRO_SPEECH_INCLUDES := $(INCLUDES) -I./examples/micro_speech
+
+micro_speech: libtflm
+	@mkdir -p $(BINDIR)
+	$(CXX) $(CXXFLAGS) $(MICRO_SPEECH_SRCS) $(MICRO_SPEECH_THIRD_PARTY_SRCS) $(MICRO_SPEECH_INCLUDES) $(LIB) -o $(BINDIR)/$@
+
+PERSON_DETECTION_SRCS := $(wildcard examples/person_detection/*.cc)
+PERSON_DETECTION_THIRD_PARTY_SRCS := $(wildcard third_party/person_model_int8/*.cc)
+PERSON_DETECTION_INCLUDES := $(INCLUDES) -I./examples/person_detection
+
+person_detection: libtflm
+	@mkdir -p $(BINDIR)
+	$(CXX) $(CXXFLAGS) $(PERSON_DETECTION_SRCS) $(PERSON_DETECTION_THIRD_PARTY_SRCS) $(PERSON_DETECTION_INCLUDES) $(LIB) -o $(BINDIR)/$@
+
+examples: hello_world magic_wand micro_speech person_detection

tensorflow/lite/micro/tools/project_generation/create_tflm_tree.py

@@ -32,6 +32,7 @@ we get further along in our prototyping. See this github issue for more details:
 import argparse
 import fileinput
 import os
+import re
 import shutil
 import subprocess
 
@@ -67,11 +68,11 @@ def _third_party_src_and_dest_files(prefix_dir, makefile_options):
                                   makefile_options))
 
   # The list_third_party_* rules give path relative to the root of the git repo.
-  # However, in the output tree, we would like for the third_party code to be a tree
-  # under prefix_dir/third_party, with the path to the tflm_download directory
-  # removed. The path manipulation logic that follows removes the downloads
-  # directory prefix, and adds the third_party prefix to create a list of
-  # destination directories for each of the third party files.
+  # However, in the output tree, we would like for the third_party code to be a
+  # tree under prefix_dir/third_party, with the path to the tflm_download
+  # directory removed. The path manipulation logic that follows removes the
+  # downloads directory prefix, and adds the third_party prefix to create a
+  # list of destination directories for each of the third party files.
   tflm_download_path = "tensorflow/lite/micro/tools/make/downloads"
   dest_files = [
       os.path.join(prefix_dir, "third_party",
@@ -125,11 +126,25 @@ def _create_examples_tree(prefix_dir, examples_list):
   # examples are in tensorflow/lite/micro/examples). However, in the output
   # tree, we would like for the examples to be under prefix_dir/examples.
   tflm_examples_path = "tensorflow/lite/micro/examples"
-
-  dest_file_list = [
-      os.path.join(prefix_dir, "examples",
-                   os.path.relpath(f, tflm_examples_path)) for f in files
-  ]
+  tflm_downloads_path = "tensorflow/lite/micro/tools/make/downloads"
+
+  # Some non-example source and headers will be in the {files} list. They need
+  # special handling or they will end up outside the {prefix_dir} tree.
+  dest_file_list = []
+  for f in files:
+    if tflm_examples_path in f:
+      # file is in examples tree
+      relative_path = os.path.relpath(f, tflm_examples_path)
+      full_filename = os.path.join(prefix_dir, "examples", relative_path)
+    elif tflm_downloads_path in f:
+      # is third-party file
+      relative_path = os.path.relpath(f, tflm_downloads_path)
+      full_filename = os.path.join(prefix_dir, "third_party", relative_path)
+    else:
+      # not third-party and not examples, don't modify file name
+      # ex. tensorflow/lite/experimental/microfrontend
+      full_filename = os.path.join(prefix_dir, f)
+    dest_file_list.append(full_filename)
 
   for dest_file, filepath in zip(dest_file_list, files):
     dest_dir = os.path.dirname(dest_file)
@@ -139,19 +154,22 @@ def _create_examples_tree(prefix_dir, examples_list):
   # Since we are changing the directory structure for the examples, we will also
   # need to modify the paths in the code.
   for filepath in dest_file_list:
-    # We need a trailing forward slash because what we care about is replacing
-    # the include paths.
-    text_to_replace = os.path.join(
-        tflm_examples_path, os.path.basename(os.path.dirname(filepath))) + "/"
-
     with fileinput.FileInput(filepath, inplace=True) as f:
       for line in f:
+        include_match = re.match(
+            r'.*#include.*"' + tflm_examples_path + r'/([^/]+)/.*"', line)
+        if include_match:
+          # We need a trailing forward slash because what we care about is
+          # replacing the include paths.
+          text_to_replace = os.path.join(tflm_examples_path,
+                                         include_match.group(1)) + "/"
+          line = line.replace(text_to_replace, "")
         # end="" prevents an extra newline from getting added as part of the
         # in-place find and replace.
-        print(line.replace(text_to_replace, ""), end="")
+        print(line, end="")
 
 
-if __name__ == "__main__":
+def main():
   parser = argparse.ArgumentParser(
       description="Starting script for TFLM project generation")
   parser.add_argument("output_dir",
@@ -197,7 +215,7 @@ if __name__ == "__main__":
     process = subprocess.Popen(params_list,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
-    stdout, stderr = process.communicate()
+    _, stderr = process.communicate()
     if process.returncode != 0:
       raise RuntimeError("%s failed with \n\n %s" %
                          (" ".join(params_list), stderr.decode()))
@@ -216,3 +234,7 @@ if __name__ == "__main__":
 
   if args.examples is not None:
     _create_examples_tree(args.output_dir, args.examples)
+
+
+if __name__ == "__main__":
+  main()