Refactor PlatformIO build scripts (#6887)

tools/platformio-build-esp32.py

@@ -24,15 +24,13 @@ http://arduino.cc/en/Reference/HomePage
 
 # Extends: https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py
 
-from os.path import abspath, basename, isdir, isfile, join
+from os.path import basename, join
 
 from SCons.Script import DefaultEnvironment
 
 env = DefaultEnvironment()
-platform = env.PioPlatform()
 
-FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
-assert isdir(FRAMEWORK_DIR)
+FRAMEWORK_DIR = env.PioPlatform().get_package_dir("framework-arduinoespressif32")
 
 env.Append(
     ASFLAGS=[
@@ -320,71 +318,5 @@ env.Append(
         ("ARDUINO_BOARD", '\\"%s\\"' % env.BoardConfig().get("name").replace('"', "")),
         "ARDUINO_PARTITION_%s" % basename(env.BoardConfig().get(
             "build.partitions", "default.csv")).replace(".csv", "").replace("-", "_")
-    ],
-
-    LIBSOURCE_DIRS=[
-        join(FRAMEWORK_DIR, "libraries")
-    ],
-
-    FLASH_EXTRA_IMAGES=[
-        ("0x1000", join(FRAMEWORK_DIR, "tools", "sdk", "esp32", "bin", "bootloader_${BOARD_FLASH_MODE}_${__get_board_f_flash(__env__)}.bin")),
-        ("0x8000", join(env.subst("$BUILD_DIR"), "partitions.bin")),
-        ("0xe000", join(FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin"))
     ]
-    + [
-        (offset, join(FRAMEWORK_DIR, img))
-        for offset, img in env.BoardConfig().get(
-            "upload.arduino.flash_extra_images", []
-        )
-    ],
 )
-
-#
-# Target: Build Core Library
-#
-
-libs = []
-
-variants_dir = join(FRAMEWORK_DIR, "variants")
-
-if "build.variants_dir" in env.BoardConfig():
-    variants_dir = join("$PROJECT_DIR", env.BoardConfig().get("build.variants_dir"))
-
-if "build.variant" in env.BoardConfig():
-    env.Append(
-        CPPPATH=[
-            join(variants_dir, env.BoardConfig().get("build.variant"))
-        ]
-    )
-    env.BuildSources(
-        join("$BUILD_DIR", "FrameworkArduinoVariant"),
-        join(variants_dir, env.BoardConfig().get("build.variant"))
-    )
-
-envsafe = env.Clone()
-
-libs.append(envsafe.BuildLibrary(
-    join("$BUILD_DIR", "FrameworkArduino"),
-    join(FRAMEWORK_DIR, "cores", env.BoardConfig().get("build.core"))
-))
-
-env.Prepend(LIBS=libs)
-
-#
-# Generate partition table
-#
-
-fwpartitions_dir = join(FRAMEWORK_DIR, "tools", "partitions")
-partitions_csv = env.BoardConfig().get("build.partitions", "default.csv")
-env.Replace(
-    PARTITIONS_TABLE_CSV=abspath(
-        join(fwpartitions_dir, partitions_csv) if isfile(
-            join(fwpartitions_dir, partitions_csv)) else partitions_csv))
-
-partition_table = env.Command(
-    join("$BUILD_DIR", "partitions.bin"),
-    "$PARTITIONS_TABLE_CSV",
-    env.VerboseAction('"$PYTHONEXE" "%s" -q $SOURCE $TARGET' % join(
-        FRAMEWORK_DIR, "tools", "gen_esp32part.py"),
-                      "Generating partitions $TARGET"))
-env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", partition_table)

tools/platformio-build-esp32c3.py

@@ -24,15 +24,13 @@ http://arduino.cc/en/Reference/HomePage
 
 # Extends: https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py
 
-from os.path import abspath, basename, isdir, isfile, join
+from os.path import basename, join
 
 from SCons.Script import DefaultEnvironment
 
 env = DefaultEnvironment()
-platform = env.PioPlatform()
 
-FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
-assert isdir(FRAMEWORK_DIR)
+FRAMEWORK_DIR = env.PioPlatform().get_package_dir("framework-arduinoespressif32")
 
 env.Append(
     ASFLAGS=[
@@ -313,71 +311,5 @@ env.Append(
         ("ARDUINO_BOARD", '\\"%s\\"' % env.BoardConfig().get("name").replace('"', "")),
         "ARDUINO_PARTITION_%s" % basename(env.BoardConfig().get(
             "build.partitions", "default.csv")).replace(".csv", "").replace("-", "_")
-    ],
-
-    LIBSOURCE_DIRS=[
-        join(FRAMEWORK_DIR, "libraries")
-    ],
-
-    FLASH_EXTRA_IMAGES=[
-        ("0x0000", join(FRAMEWORK_DIR, "tools", "sdk", "esp32c3", "bin", "bootloader_${BOARD_FLASH_MODE}_${__get_board_f_flash(__env__)}.bin")),
-        ("0x8000", join(env.subst("$BUILD_DIR"), "partitions.bin")),
-        ("0xe000", join(FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin"))
     ]
-    + [
-        (offset, join(FRAMEWORK_DIR, img))
-        for offset, img in env.BoardConfig().get(
-            "upload.arduino.flash_extra_images", []
-        )
-    ],
 )
-
-#
-# Target: Build Core Library
-#
-
-libs = []
-
-variants_dir = join(FRAMEWORK_DIR, "variants")
-
-if "build.variants_dir" in env.BoardConfig():
-    variants_dir = join("$PROJECT_DIR", env.BoardConfig().get("build.variants_dir"))
-
-if "build.variant" in env.BoardConfig():
-    env.Append(
-        CPPPATH=[
-            join(variants_dir, env.BoardConfig().get("build.variant"))
-        ]
-    )
-    env.BuildSources(
-        join("$BUILD_DIR", "FrameworkArduinoVariant"),
-        join(variants_dir, env.BoardConfig().get("build.variant"))
-    )
-
-envsafe = env.Clone()
-
-libs.append(envsafe.BuildLibrary(
-    join("$BUILD_DIR", "FrameworkArduino"),
-    join(FRAMEWORK_DIR, "cores", env.BoardConfig().get("build.core"))
-))
-
-env.Prepend(LIBS=libs)
-
-#
-# Generate partition table
-#
-
-fwpartitions_dir = join(FRAMEWORK_DIR, "tools", "partitions")
-partitions_csv = env.BoardConfig().get("build.partitions", "default.csv")
-env.Replace(
-    PARTITIONS_TABLE_CSV=abspath(
-        join(fwpartitions_dir, partitions_csv) if isfile(
-            join(fwpartitions_dir, partitions_csv)) else partitions_csv))
-
-partition_table = env.Command(
-    join("$BUILD_DIR", "partitions.bin"),
-    "$PARTITIONS_TABLE_CSV",
-    env.VerboseAction('"$PYTHONEXE" "%s" -q $SOURCE $TARGET' % join(
-        FRAMEWORK_DIR, "tools", "gen_esp32part.py"),
-                      "Generating partitions $TARGET"))
-env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", partition_table)

tools/platformio-build-esp32s2.py

@@ -24,15 +24,14 @@ http://arduino.cc/en/Reference/HomePage
 
 # Extends: https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py
 
-from os.path import abspath, basename, isdir, isfile, join
+from os.path import basename, join
 
 from SCons.Script import DefaultEnvironment
 
 env = DefaultEnvironment()
-platform = env.PioPlatform()
 
-FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
-assert isdir(FRAMEWORK_DIR)
+FRAMEWORK_DIR = env.PioPlatform().get_package_dir("framework-arduinoespressif32")
+
 
 env.Append(
     ASFLAGS=[
@@ -315,71 +314,5 @@ env.Append(
         ("ARDUINO_BOARD", '\\"%s\\"' % env.BoardConfig().get("name").replace('"', "")),
         "ARDUINO_PARTITION_%s" % basename(env.BoardConfig().get(
             "build.partitions", "default.csv")).replace(".csv", "").replace("-", "_")
-    ],
-
-    LIBSOURCE_DIRS=[
-        join(FRAMEWORK_DIR, "libraries")
-    ],
-
-    FLASH_EXTRA_IMAGES=[
-        ("0x1000", join(FRAMEWORK_DIR, "tools", "sdk", "esp32s2", "bin", "bootloader_${BOARD_FLASH_MODE}_${__get_board_f_flash(__env__)}.bin")),
-        ("0x8000", join(env.subst("$BUILD_DIR"), "partitions.bin")),
-        ("0xe000", join(FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin"))
     ]
-    + [
-        (offset, join(FRAMEWORK_DIR, img))
-        for offset, img in env.BoardConfig().get(
-            "upload.arduino.flash_extra_images", []
-        )
-    ],
 )
-
-#
-# Target: Build Core Library
-#
-
-libs = []
-
-variants_dir = join(FRAMEWORK_DIR, "variants")
-
-if "build.variants_dir" in env.BoardConfig():
-    variants_dir = join("$PROJECT_DIR", env.BoardConfig().get("build.variants_dir"))
-
-if "build.variant" in env.BoardConfig():
-    env.Append(
-        CPPPATH=[
-            join(variants_dir, env.BoardConfig().get("build.variant"))
-        ]
-    )
-    env.BuildSources(
-        join("$BUILD_DIR", "FrameworkArduinoVariant"),
-        join(variants_dir, env.BoardConfig().get("build.variant"))
-    )
-
-envsafe = env.Clone()
-
-libs.append(envsafe.BuildLibrary(
-    join("$BUILD_DIR", "FrameworkArduino"),
-    join(FRAMEWORK_DIR, "cores", env.BoardConfig().get("build.core"))
-))
-
-env.Prepend(LIBS=libs)
-
-#
-# Generate partition table
-#
-
-fwpartitions_dir = join(FRAMEWORK_DIR, "tools", "partitions")
-partitions_csv = env.BoardConfig().get("build.partitions", "default.csv")
-env.Replace(
-    PARTITIONS_TABLE_CSV=abspath(
-        join(fwpartitions_dir, partitions_csv) if isfile(
-            join(fwpartitions_dir, partitions_csv)) else partitions_csv))
-
-partition_table = env.Command(
-    join("$BUILD_DIR", "partitions.bin"),
-    "$PARTITIONS_TABLE_CSV",
-    env.VerboseAction('"$PYTHONEXE" "%s" -q $SOURCE $TARGET' % join(
-        FRAMEWORK_DIR, "tools", "gen_esp32part.py"),
-                      "Generating partitions $TARGET"))
-env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", partition_table)

tools/platformio-build-esp32s3.py

@@ -24,15 +24,13 @@ http://arduino.cc/en/Reference/HomePage
 
 # Extends: https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py
 
-from os.path import abspath, basename, isdir, isfile, join
+from os.path import basename, join
 
 from SCons.Script import DefaultEnvironment
 
 env = DefaultEnvironment()
-platform = env.PioPlatform()
 
-FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
-assert isdir(FRAMEWORK_DIR)
+FRAMEWORK_DIR = env.PioPlatform().get_package_dir("framework-arduinoespressif32")
 
 env.Append(
     ASFLAGS=[
@@ -332,71 +330,5 @@ env.Append(
         ("ARDUINO_BOARD", '\\"%s\\"' % env.BoardConfig().get("name").replace('"', "")),
         "ARDUINO_PARTITION_%s" % basename(env.BoardConfig().get(
             "build.partitions", "default.csv")).replace(".csv", "").replace("-", "_")
-    ],
-
-    LIBSOURCE_DIRS=[
-        join(FRAMEWORK_DIR, "libraries")
-    ],
-
-    FLASH_EXTRA_IMAGES=[
-        ("0x0000", join(FRAMEWORK_DIR, "tools", "sdk", "esp32s3", "bin", "bootloader_${BOARD_FLASH_MODE}_${__get_board_f_flash(__env__)}.bin")),
-        ("0x8000", join(env.subst("$BUILD_DIR"), "partitions.bin")),
-        ("0xe000", join(FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin"))
     ]
-    + [
-        (offset, join(FRAMEWORK_DIR, img))
-        for offset, img in env.BoardConfig().get(
-            "upload.arduino.flash_extra_images", []
-        )
-    ],
 )
-
-#
-# Target: Build Core Library
-#
-
-libs = []
-
-variants_dir = join(FRAMEWORK_DIR, "variants")
-
-if "build.variants_dir" in env.BoardConfig():
-    variants_dir = join("$PROJECT_DIR", env.BoardConfig().get("build.variants_dir"))
-
-if "build.variant" in env.BoardConfig():
-    env.Append(
-        CPPPATH=[
-            join(variants_dir, env.BoardConfig().get("build.variant"))
-        ]
-    )
-    env.BuildSources(
-        join("$BUILD_DIR", "FrameworkArduinoVariant"),
-        join(variants_dir, env.BoardConfig().get("build.variant"))
-    )
-
-envsafe = env.Clone()
-
-libs.append(envsafe.BuildLibrary(
-    join("$BUILD_DIR", "FrameworkArduino"),
-    join(FRAMEWORK_DIR, "cores", env.BoardConfig().get("build.core"))
-))
-
-env.Prepend(LIBS=libs)
-
-#
-# Generate partition table
-#
-
-fwpartitions_dir = join(FRAMEWORK_DIR, "tools", "partitions")
-partitions_csv = env.BoardConfig().get("build.partitions", "default.csv")
-env.Replace(
-    PARTITIONS_TABLE_CSV=abspath(
-        join(fwpartitions_dir, partitions_csv) if isfile(
-            join(fwpartitions_dir, partitions_csv)) else partitions_csv))
-
-partition_table = env.Command(
-    join("$BUILD_DIR", "partitions.bin"),
-    "$PARTITIONS_TABLE_CSV",
-    env.VerboseAction('"$PYTHONEXE" "%s" -q $SOURCE $TARGET' % join(
-        FRAMEWORK_DIR, "tools", "gen_esp32part.py"),
-                      "Generating partitions $TARGET"))
-env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", partition_table)

tools/platformio-build.py

@@ -24,13 +24,70 @@ http://arduino.cc/en/Reference/HomePage
 
 # Extends: https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py
 
-from os.path import join
+from os.path import abspath, isdir, isfile, join
 
 from SCons.Script import DefaultEnvironment, SConscript
 
 env = DefaultEnvironment()
-board = env.BoardConfig()
-build_mcu = board.get("build.mcu", "").lower()
+platform = env.PioPlatform()
+board_config = env.BoardConfig()
+build_mcu = board_config.get("build.mcu", "").lower()
+
+FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
+assert isdir(FRAMEWORK_DIR)
+
+
+#
+# Helpers
+#
+
+def get_partition_table_csv(variants_dir):
+    fwpartitions_dir = join(FRAMEWORK_DIR, "tools", "partitions")
+
+    custom_partition = board_config.get(
+        "build.partitions", board_config.get("build.arduino.partitions", "")
+    )
+
+    if custom_partition:
+        partitions_csv = board_config.get("build.partitions", board_config.get(
+            "build.arduino.partitions", "default.csv"))
+        return abspath(
+            join(fwpartitions_dir, partitions_csv)
+            if isfile(join(fwpartitions_dir, partitions_csv))
+            else partitions_csv
+        )
+
+    variant_partitions = join(
+        variants_dir, board_config.get("build.variant", ""), "partitions.csv"
+    )
+    return (
+        variant_partitions
+        if isfile(variant_partitions)
+        else join(fwpartitions_dir, "default.csv")
+    )
+
+
+def get_bootloader_image(variants_dir):
+    variant_bootloader = join(
+        variants_dir, board_config.get("build.variant", ""), "bootloader.bin"
+    )
+    return (
+        variant_bootloader
+        if isfile(variant_bootloader)
+        else join(
+            FRAMEWORK_DIR,
+            "tools",
+            "sdk",
+            build_mcu,
+            "bin",
+            "bootloader_${__get_board_boot_mode(__env__)}_${__get_board_f_flash(__env__)}.bin",
+        )
+    )
+
+
+#
+# Run target-specific script to populate the environment with proper build flags
+#
 
 SConscript(
     join(
@@ -41,3 +98,75 @@ SConscript(
         "platformio-build-%s.py" % build_mcu,
     )
 )
+
+#
+# Process framework extra images
+#
+
+env.Append(
+    LIBSOURCE_DIRS=[
+        join(FRAMEWORK_DIR, "libraries")
+    ],
+
+    FLASH_EXTRA_IMAGES=[
+        (
+            "0x1000" if build_mcu in ("esp32", "esp32s2") else "0x0000",
+            get_bootloader_image(board_config.get(
+                "build.variants_dir", join(FRAMEWORK_DIR, "variants")))
+        ),
+        ("0x8000", join(env.subst("$BUILD_DIR"), "partitions.bin")),
+        ("0xe000", join(FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin"))
+    ]
+    + [
+        (offset, join(FRAMEWORK_DIR, img))
+        for offset, img in board_config.get(
+            "upload.arduino.flash_extra_images", []
+        )
+    ],
+)
+
+#
+# Target: Build Core Library
+#
+
+libs = []
+
+variants_dir = join(FRAMEWORK_DIR, "variants")
+
+if "build.variants_dir" in board_config:
+    variants_dir = join("$PROJECT_DIR", board_config.get("build.variants_dir"))
+
+if "build.variant" in board_config:
+    env.Append(
+        CPPPATH=[
+            join(variants_dir, board_config.get("build.variant"))
+        ]
+    )
+    env.BuildSources(
+        join("$BUILD_DIR", "FrameworkArduinoVariant"),
+        join(variants_dir, board_config.get("build.variant"))
+    )
+
+libs.append(env.BuildLibrary(
+    join("$BUILD_DIR", "FrameworkArduino"),
+    join(FRAMEWORK_DIR, "cores", board_config.get("build.core"))
+))
+
+env.Prepend(LIBS=libs)
+
+#
+# Generate partition table
+#
+
+env.Replace(PARTITIONS_TABLE_CSV=get_partition_table_csv(variants_dir))
+
+partition_table = env.Command(
+    join("$BUILD_DIR", "partitions.bin"),
+    "$PARTITIONS_TABLE_CSV",
+    env.VerboseAction(
+        '"$PYTHONEXE" "%s" -q $SOURCE $TARGET'
+        % join(FRAMEWORK_DIR, "tools", "gen_esp32part.py"),
+        "Generating partitions $TARGET",
+    ),
+)
+env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", partition_table)