Merge tag 'dm-pull-9jul19-take2' of https://gitlab.denx.de/u-boot/custodians/u-boot-dm

- Sandbox improvements including .dts refactor
- Minor tracing and PCI improvements
- Various other minor fixes
- Conversion of patman, dtoc and binman to support Python 3

arch/sandbox/config.mk

@@ -5,14 +5,15 @@ PLATFORM_CPPFLAGS += -D__SANDBOX__ -U_FORTIFY_SOURCE
 PLATFORM_CPPFLAGS += -DCONFIG_ARCH_MAP_SYSMEM
 PLATFORM_CPPFLAGS += -fPIC
 PLATFORM_LIBS += -lrt
+SDL_CONFIG ?= sdl-config
 
 # Define this to avoid linking with SDL, which requires SDL libraries
 # This can solve 'sdl-config: Command not found' errors
 ifneq ($(NO_SDL),)
 PLATFORM_CPPFLAGS += -DSANDBOX_NO_SDL
 else
-PLATFORM_LIBS += $(shell sdl-config --libs)
-PLATFORM_CPPFLAGS += $(shell sdl-config --cflags)
+PLATFORM_LIBS += $(shell $(SDL_CONFIG) --libs)
+PLATFORM_CPPFLAGS += $(shell $(SDL_CONFIG) --cflags)
 endif
 
 cmd_u-boot__ = $(CC) -o $@ -Wl,-T u-boot.lds $(u-boot-init) \

arch/sandbox/cpu/spl.c

@@ -12,6 +12,7 @@
 
 DECLARE_GLOBAL_DATA_PTR;
 
+/* SPL / TPL init function */
 void board_init_f(ulong flag)
 {
 	struct sandbox_state *state = state_get_current();
@@ -44,7 +45,7 @@ static int spl_board_load_image(struct spl_image_info *spl_image,
 
 	return 0;
 }
-SPL_LOAD_IMAGE_METHOD("sandbox", 0, BOOT_DEVICE_BOARD, spl_board_load_image);
+SPL_LOAD_IMAGE_METHOD("sandbox", 9, BOOT_DEVICE_BOARD, spl_board_load_image);
 
 void spl_board_init(void)
 {

arch/sandbox/cpu/state.c

@@ -356,6 +356,7 @@ void state_reset_for_test(struct sandbox_state *state)
 	/* No reset yet, so mark it as such. Always allow power reset */
 	state->last_sysreset = SYSRESET_COUNT;
 	state->sysreset_allowed[SYSRESET_POWER] = true;
+	state->sysreset_allowed[SYSRESET_POWER_OFF] = true;
 
 	memset(&state->wdt, '\0', sizeof(state->wdt));
 	memset(state->spi, '\0', sizeof(state->spi));

arch/sandbox/dts/sandbox.dts

 /dts-v1/;
 
-#define USB_CLASS_HUB			9
+#include <config.h>
 
 / {
 	#address-cells = <1>;
@@ -12,46 +12,17 @@
 		pci0 = &pci;
 		rtc0 = &rtc_0;
 		axi0 = &axi;
+		spi0 = &spi;
 	};
 
-	chosen {
-		stdout-path = "/serial";
-	};
-
-	audio: audio-codec {
-		compatible = "sandbox,audio-codec";
-		#sound-dai-cells = <1>;
+	memory {
+		reg = <0 CONFIG_SYS_SDRAM_SIZE>;
 	};
 
 	cros_ec: cros-ec {
 		reg = <0 0>;
 		u-boot,dm-pre-reloc;
 		compatible = "google,cros-ec-sandbox";
-
-		/*
-		 * This describes the flash memory within the EC. Note
-		 * that the STM32L flash erases to 0, not 0xff.
-		 */
-		flash {
-			u-boot,dm-pre-reloc;
-			image-pos = <0x08000000>;
-			size = <0x20000>;
-			erase-value = <0>;
-
-			/* Information for sandbox */
-			ro {
-				image-pos = <0>;
-				size = <0xf000>;
-			};
-			wp-ro {
-				image-pos = <0xf000>;
-				size = <0x1000>;
-			};
-			rw {
-				image-pos = <0x10000>;
-				size = <0x10000>;
-			};
-		};
 	};
 
 	ethrawbus {
@@ -65,30 +36,6 @@
 		fake-host-hwaddr = [00 00 66 44 22 00];
 	};
 
-	gpio_a: gpios@0 {
-		u-boot,dm-pre-reloc;
-		gpio-controller;
-		compatible = "sandbox,gpio";
-		#gpio-cells = <1>;
-		gpio-bank-name = "a";
-		sandbox,gpio-count = <20>;
-	};
-
-	gpio_b: gpios@1 {
-		u-boot,dm-pre-reloc;
-		gpio-controller;
-		compatible = "sandbox,gpio";
-		#gpio-cells = <2>;
-		gpio-bank-name = "b";
-		sandbox,gpio-count = <10>;
-	};
-
-	hexagon {
-		compatible = "demo-simple";
-		colour = "white";
-		sides = <6>;
-	};
-
 	i2c_0: i2c@0 {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -97,63 +44,6 @@
 		clock-frequency = <400000>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_i2c0>;
-		eeprom@2c {
-			reg = <0x2c>;
-			compatible = "i2c-eeprom";
-			sandbox,emul = <&emul_eeprom>;
-		};
-
-		rtc_0: rtc@43 {
-			reg = <0x43>;
-			compatible = "sandbox-rtc";
-			sandbox,emul = <&emul0>;
-		};
-		sandbox_pmic: sandbox_pmic {
-			reg = <0x40>;
-		};
-
-		mc34708: pmic@41 {
-			reg = <0x41>;
-		};
-
-		i2c_emul: emul {
-			reg = <0xff>;
-			compatible = "sandbox,i2c-emul-parent";
-			emul_eeprom: emul-eeprom {
-				compatible = "sandbox,i2c-eeprom";
-				sandbox,filename = "i2c.bin";
-				sandbox,size = <256>;
-			};
-			emul0: emul0 {
-				compatible = "sandbox,i2c-rtc";
-			};
-		};
-	};
-
-	i2s: i2s {
-		compatible = "sandbox,i2s";
-		#sound-dai-cells = <1>;
-	};
-
-	lcd {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,lcd-sdl";
-		xres = <1366>;
-		yres = <768>;
-	};
-
-	leds {
-		compatible = "gpio-leds";
-
-		iracibble {
-			gpios = <&gpio_a 1 0>;
-			label = "sandbox:red";
-		};
-
-		martinet {
-			gpios = <&gpio_a 2 0>;
-			label = "sandbox:green";
-		};
 	};
 
 	pci: pci-controller {
@@ -163,233 +53,19 @@
 		#size-cells = <2>;
 		ranges = <0x02000000 0 0x10000000 0x10000000 0 0x2000
 				0x01000000 0 0x20000000 0x20000000 0 0x2000>;
-		pci@1f,0 {
-			compatible = "pci-generic";
-			reg = <0xf800 0 0 0 0>;
-			emul@1f,0 {
-				compatible = "sandbox,swap-case";
-			};
-		};
-	};
-
-	pinctrl {
-		compatible = "sandbox,pinctrl";
-		status = "okay";
-
-		pinctrl_i2c0: i2c0 {
-			groups = "i2c";
-			function = "i2c";
-			bias-pull-up;
-		};
-
-		pinctrl_serial0: uart0 {
-			groups = "serial_a";
-			function = "serial";
 	};
 
-		pinctrl_onewire0: onewire0 {
-			groups = "w1";
-			function = "w1";
-			bias-pull-up;
-		};
-	};
-
-	reset@1 {
-		compatible = "sandbox,reset";
-	};
-
-	sound {
-		compatible = "sandbox,sound";
-		cpu {
-			sound-dai = <&i2s 0>;
-		};
-
-		codec {
-			sound-dai = <&audio 0>;
-		};
-	};
-
-	spi@0 {
+	spi: spi@0 {
 		u-boot,dm-pre-reloc;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		reg = <0 0>;
 		compatible = "sandbox,spi";
 		cs-gpios = <0>, <&gpio_a 0>;
-		firmware_storage_spi: flash@0 {
-			u-boot,dm-pre-reloc;
-			reg = <0>;
-			compatible = "spansion,m25p16", "sandbox,spi-flash";
-			spi-max-frequency = <40000000>;
-			sandbox,filename = "spi.bin";
-		};
 	};
 
-	spl-test {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,spl-test";
-		boolval;
-		intval = <1>;
-		intarray = <2 3 4>;
-		byteval = [05];
-		bytearray = [06];
-		longbytearray = [09 0a 0b 0c 0d 0e 0f 10 11];
-		stringval = "message";
-		stringarray = "multi-word", "message";
-	};
-
-	spl-test2 {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,spl-test";
-		intval = <3>;
-		intarray = <5>;
-		byteval = [08];
-		bytearray = [01 23 34];
-		longbytearray = [09 0a 0b 0c];
-		stringval = "message2";
-		stringarray = "another", "multi-word", "message";
-	};
-
-	spl-test3 {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,spl-test";
-		stringarray = "one";
-	};
-
-	spl-test4 {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,spl-test.2";
-	};
-
-	square {
-		compatible = "demo-shape";
-		colour = "blue";
-		sides = <4>;
-	};
-
-	timer {
-		compatible = "sandbox,timer";
-		clock-frequency = <1000000>;
-	};
-
-	tpm {
-		u-boot,dm-pre-reloc;
-		compatible = "google,sandbox-tpm";
-	};
-
-	tpm2 {
-		compatible = "sandbox,tpm2";
-	};
-
-	triangle {
-		compatible = "demo-shape";
-		colour = "cyan";
-		sides = <3>;
-		character = <83>;
-		light-gpios = <&gpio_a 2>, <&gpio_b 6 0>;
-	};
-
-	/* Needs to be available prior to relocation */
-	uart0: serial {
-		u-boot,dm-spl;
-		compatible = "sandbox,serial";
-		sandbox,text-colour = "cyan";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_serial0>;
-	};
-
-	usb@0 {
-		compatible = "sandbox,usb";
-		status = "disabled";
-		hub {
-			compatible = "sandbox,usb-hub";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			flash-stick {
-				reg = <0>;
-				compatible = "sandbox,usb-flash";
-			};
-		};
-	};
-
-	usb@1 {
-		compatible = "sandbox,usb";
-		hub {
-			compatible = "usb-hub";
-			usb,device-class = <USB_CLASS_HUB>;
-			hub-emul {
-				compatible = "sandbox,usb-hub";
-				#address-cells = <1>;
-				#size-cells = <0>;
-				flash-stick {
-					reg = <0>;
-					compatible = "sandbox,usb-flash";
-					sandbox,filepath = "flash.bin";
-				};
-			};
-		};
-	};
-
-	usb@2 {
-		compatible = "sandbox,usb";
-		status = "disabled";
-	};
-
-	spmi: spmi@0 {
-		compatible = "sandbox,spmi";
-		#address-cells = <0x1>;
-		#size-cells = <0x1>;
-		pm8916@0 {
-			compatible = "qcom,spmi-pmic";
-			reg = <0x0 0x1>;
-			#address-cells = <0x1>;
-			#size-cells = <0x1>;
-
-			spmi_gpios: gpios@c000 {
-				compatible = "qcom,pm8916-gpio";
-				reg = <0xc000 0x400>;
-				gpio-controller;
-				gpio-count = <4>;
-				#gpio-cells = <2>;
-				gpio-bank-name="spmi";
-			};
-		};
-	};
-
-	axi: axi@0 {
-		compatible = "sandbox,axi";
-		#address-cells = <0x1>;
-		#size-cells = <0x1>;
-		store@0 {
-			compatible = "sandbox,sandbox_store";
-			reg = <0x0 0x400>;
-		};
-	};
-
-	onewire0: onewire {
-		compatible = "w1-gpio";
-		gpios = <&gpio_a 8>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_onewire0>;
-		status = "okay";
-
-		sandbox_eeprom0: sandbox_eeprom@0 {
-			compatible = "sandbox,w1-eeprom";
-			status = "okay";
-		};
-	};
-
-	sandbox_tee {
-		compatible = "sandbox,tee";
-	};
 };
 
+#include "sandbox.dtsi"
 #include "cros-ec-keyboard.dtsi"
 #include "sandbox_pmic.dtsi"
-
-&cros_ec {
-	u-boot,dm-pre-reloc;
-	keyboard-controller {
-		u-boot,dm-pre-reloc;
-	};
-};

arch/sandbox/dts/sandbox.dtsi

+/*
+ * This is the common sandbox device-tree nodes. This is shared between sandbox
+ * and sandbox64 builds.
+ */
+
+#define USB_CLASS_HUB			9
+
+/ {
+	chosen {
+		stdout-path = "/serial";
+	};
+
+	audio: audio-codec {
+		compatible = "sandbox,audio-codec";
+		#sound-dai-cells = <1>;
+	};
+
+	gpio_a: gpios@0 {
+		u-boot,dm-pre-reloc;
+		gpio-controller;
+		compatible = "sandbox,gpio";
+		#gpio-cells = <1>;
+		gpio-bank-name = "a";
+		sandbox,gpio-count = <20>;
+	};
+
+	gpio_b: gpios@1 {
+		u-boot,dm-pre-reloc;
+		gpio-controller;
+		compatible = "sandbox,gpio";
+		#gpio-cells = <2>;
+		gpio-bank-name = "b";
+		sandbox,gpio-count = <10>;
+	};
+
+	hexagon {
+		compatible = "demo-simple";
+		colour = "white";
+		sides = <6>;
+	};
+
+	i2c_0: i2c@0 {
+		eeprom@2c {
+			reg = <0x2c>;
+			compatible = "i2c-eeprom";
+			sandbox,emul = <&emul_eeprom>;
+		};
+
+		rtc_0: rtc@43 {
+			reg = <0x43>;
+			compatible = "sandbox-rtc";
+			sandbox,emul = <&emul0>;
+		};
+		sandbox_pmic: sandbox_pmic {
+			reg = <0x40>;
+		};
+
+		mc34708: pmic@41 {
+			reg = <0x41>;
+		};
+
+		i2c_emul: emul {
+			reg = <0xff>;
+			compatible = "sandbox,i2c-emul-parent";
+			emul_eeprom: emul-eeprom {
+				compatible = "sandbox,i2c-eeprom";
+				sandbox,filename = "i2c.bin";
+				sandbox,size = <256>;
+			};
+			emul0: emul0 {
+				compatible = "sandbox,i2c-rtc";
+			};
+		};
+	};
+
+	i2s: i2s {
+		compatible = "sandbox,i2s";
+		#sound-dai-cells = <1>;
+	};
+
+	lcd {
+		u-boot,dm-pre-reloc;
+		compatible = "sandbox,lcd-sdl";
+		xres = <1366>;
+		yres = <768>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		iracibble {
+			gpios = <&gpio_a 1 0>;
+			label = "sandbox:red";
+		};
+
+		martinet {
+			gpios = <&gpio_a 2 0>;
+			label = "sandbox:green";
+		};
+	};
+
+	pci-controller {
+		pci@1f,0 {
+			compatible = "pci-generic";
+			reg = <0xf800 0 0 0 0>;
+			emul@1f,0 {
+				compatible = "sandbox,swap-case";
+			};
+		};
+	};
+
+	pinctrl {
+		compatible = "sandbox,pinctrl";
+		status = "okay";
+
+		pinctrl_i2c0: i2c0 {
+			groups = "i2c";
+			function = "i2c";
+			bias-pull-up;
+		};
+
+		pinctrl_serial0: uart0 {
+			groups = "serial_a";
+			function = "serial";
+		};
+
+		pinctrl_onewire0: onewire0 {
+			groups = "w1";
+			function = "w1";
+			bias-pull-up;
+		};
+	};
+
+	reset@1 {
+		compatible = "sandbox,reset";
+	};
+
+	sound {
+		compatible = "sandbox,sound";
+		cpu {
+			sound-dai = <&i2s 0>;
+		};
+
+		codec {
+			sound-dai = <&audio 0>;
+		};
+	};
+
+	spi@0 {
+		firmware_storage_spi: flash@0 {
+			u-boot,dm-pre-reloc;
+			reg = <0>;
+			compatible = "spansion,m25p16", "jedec,spi-nor";
+			spi-max-frequency = <40000000>;
+			sandbox,filename = "spi.bin";
+		};
+	};
+
+	spl-test {
+		u-boot,dm-pre-reloc;
+		compatible = "sandbox,spl-test";
+		boolval;
+		intval = <1>;
+		intarray = <2 3 4>;
+		byteval = [05];
+		bytearray = [06];
+		longbytearray = [09 0a 0b 0c 0d 0e 0f 10 11];
+		stringval = "message";
+		stringarray = "multi-word", "message";
+	};
+
+	spl-test2 {
+		u-boot,dm-pre-reloc;
+		compatible = "sandbox,spl-test";
+		intval = <3>;
+		intarray = <5>;
+		byteval = [08];
+		bytearray = [01 23 34];
+		longbytearray = [09 0a 0b 0c];
+		stringval = "message2";
+		stringarray = "another", "multi-word", "message";
+	};
+
+	spl-test3 {
+		u-boot,dm-pre-reloc;
+		compatible = "sandbox,spl-test";
+		stringarray = "one";
+	};
+
+	spl-test4 {
+		u-boot,dm-pre-reloc;
+		compatible = "sandbox,spl-test.2";
+	};
+
+	spl-test5 {
+		u-boot,dm-tpl;
+		compatible = "sandbox,spl-test";
+		stringarray = "tpl";
+	};
+
+	spl-test6 {
+		u-boot,dm-pre-proper;
+		compatible = "sandbox,spl-test";
+		stringarray = "pre-proper";
+	};
+
+	spl-test7 {
+		u-boot,dm-spl;
+		compatible = "sandbox,spl-test";
+		stringarray = "spl";
+	};
+
+	square {
+		compatible = "demo-shape";
+		colour = "blue";
+		sides = <4>;
+	};
+
+	timer {
+		compatible = "sandbox,timer";
+		clock-frequency = <1000000>;
+	};
+
+	tpm {
+		u-boot,dm-pre-reloc;
+		compatible = "google,sandbox-tpm";
+	};
+
+	tpm2 {
+		compatible = "sandbox,tpm2";
+	};
+
+	triangle {
+		compatible = "demo-shape";
+		colour = "cyan";
+		sides = <3>;
+		character = <83>;
+		light-gpios = <&gpio_a 2>, <&gpio_b 6 0>;
+	};
+
+	/* Needs to be available prior to relocation */
+	uart0: serial {
+		u-boot,dm-spl;
+		compatible = "sandbox,serial";
+		sandbox,text-colour = "cyan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_serial0>;
+	};
+
+	usb@0 {
+		compatible = "sandbox,usb";
+		status = "disabled";
+		hub {
+			compatible = "sandbox,usb-hub";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			flash-stick {
+				reg = <0>;
+				compatible = "sandbox,usb-flash";
+			};
+		};
+	};
+
+	usb@1 {
+		compatible = "sandbox,usb";
+		hub {
+			compatible = "usb-hub";
+			usb,device-class = <USB_CLASS_HUB>;
+			hub-emul {
+				compatible = "sandbox,usb-hub";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				flash-stick {
+					reg = <0>;
+					compatible = "sandbox,usb-flash";
+					sandbox,filepath = "flash.bin";
+				};
+			};
+		};
+	};
+
+	usb@2 {
+		compatible = "sandbox,usb";
+		status = "disabled";
+	};
+
+	spmi: spmi@0 {
+		compatible = "sandbox,spmi";
+		#address-cells = <0x1>;
+		#size-cells = <0x1>;
+		pm8916@0 {
+			compatible = "qcom,spmi-pmic";
+			reg = <0x0 0x1>;
+			#address-cells = <0x1>;
+			#size-cells = <0x1>;
+
+			spmi_gpios: gpios@c000 {
+				compatible = "qcom,pm8916-gpio";
+				reg = <0xc000 0x400>;
+				gpio-controller;
+				gpio-count = <4>;
+				#gpio-cells = <2>;
+				gpio-bank-name="spmi";
+			};
+		};
+	};
+
+	axi: axi@0 {
+		compatible = "sandbox,axi";
+		#address-cells = <0x1>;
+		#size-cells = <0x1>;
+		store@0 {
+			compatible = "sandbox,sandbox_store";
+			reg = <0x0 0x400>;
+		};
+	};
+
+	onewire0: onewire {
+		compatible = "w1-gpio";
+		gpios = <&gpio_a 8>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_onewire0>;
+		status = "okay";
+
+		sandbox_eeprom0: sandbox_eeprom@0 {
+			compatible = "sandbox,w1-eeprom";
+			status = "okay";
+		};
+	};
+
+	sandbox_tee {
+		compatible = "sandbox,tee";
+	};
+};
+
+&cros_ec {
+	/*
+	 * This describes the flash memory within the EC. Note
+	 * that the STM32L flash erases to 0, not 0xff.
+	 */
+	flash {
+		image-pos = <0x08000000>;
+		size = <0x20000>;
+		erase-value = <0>;
+
+		/* Information for sandbox */
+		ro {
+			image-pos = <0>;
+			size = <0xf000>;
+		};
+		wp-ro {
+			image-pos = <0xf000>;
+			size = <0x1000>;
+		};
+		rw {
+			image-pos = <0x10000>;
+			size = <0x10000>;
+		};
+	};
+
+	keyboard-controller {
+		u-boot,dm-pre-reloc;
+	};
+};

arch/sandbox/dts/sandbox64.dts

 /dts-v1/;
 
-#define USB_CLASS_HUB			9
+#include <config.h>
 
 / {
 	#address-cells = <2>;
@@ -11,39 +11,18 @@
 		i2c0 = &i2c_0;
 		pci0 = &pci;
 		rtc0 = &rtc_0;
+		axi0 = &axi;
+		spi0 = &spi;
 	};
 
-	chosen {
-		stdout-path = "/serial";
+	memory {
+		reg = /bits/ 64 <0 CONFIG_SYS_SDRAM_SIZE>;
 	};
 
 	cros_ec: cros-ec {
 		reg = <0 0 0 0>;
+		u-boot,dm-pre-reloc;
 		compatible = "google,cros-ec-sandbox";
-
-		/*
-		 * This describes the flash memory within the EC. Note
-		 * that the STM32L flash erases to 0, not 0xff.
-		 */
-		flash {
-			image-pos = <0x08000000>;
-			size = <0x20000>;
-			erase-value = <0>;
-
-			/* Information for sandbox */
-			ro {
-				image-pos = <0>;
-				size = <0xf000>;
-			};
-			wp-ro {
-				image-pos = <0xf000>;
-				size = <0x1000>;
-			};
-			rw {
-				image-pos = <0x10000>;
-				size = <0x10000>;
-			};
-		};
 	};
 
 	ethrawbus {
@@ -57,28 +36,6 @@
 		fake-host-hwaddr = [00 00 66 44 22 00];
 	};
 
-	gpio_a: gpios@0 {
-		gpio-controller;
-		compatible = "sandbox,gpio";
-		#gpio-cells = <1>;
-		gpio-bank-name = "a";
-		sandbox,gpio-count = <20>;
-	};
-
-	gpio_b: gpios@1 {
-		gpio-controller;
-		compatible = "sandbox,gpio";
-		#gpio-cells = <2>;
-		gpio-bank-name = "b";
-		sandbox,gpio-count = <10>;
-	};
-
-	hexagon {
-		compatible = "demo-simple";
-		colour = "white";
-		sides = <6>;
-	};
-
 	i2c_0: i2c@0 {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -87,58 +44,6 @@
 		clock-frequency = <400000>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_i2c0>;
-		eeprom@2c {
-			reg = <0x2c>;
-			compatible = "i2c-eeprom";
-			sandbox,emul = <&emul_eeprom>;
-		};
-
-		rtc_0: rtc@43 {
-			reg = <0x43>;
-			compatible = "sandbox-rtc";
-			sandbox,emul = <&emul0>;
-		};
-		sandbox_pmic: sandbox_pmic {
-			reg = <0x40>;
-		};
-
-		mc34708: pmic@41 {
-			reg = <0x41>;
-		};
-
-		i2c_emul: emul {
-			reg = <0xff>;
-			compatible = "sandbox,i2c-emul-parent";
-			emul_eeprom: emul-eeprom {
-				compatible = "sandbox,i2c-eeprom";
-				sandbox,filename = "i2c.bin";
-				sandbox,size = <256>;
-			};
-			emul0: emul0 {
-				compatible = "sandbox,i2c-rtc";
-			};
-		};
-	};
-
-	lcd {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,lcd-sdl";
-		xres = <1366>;
-		yres = <768>;
-	};
-
-	leds {
-		compatible = "gpio-leds";
-
-		iracibble {
-			gpios = <&gpio_a 1 0>;
-			label = "sandbox:red";
-		};
-
-		martinet {
-			gpios = <&gpio_a 2 0>;
-			label = "sandbox:green";
-		};
 	};
 
 	pci: pci-controller {
@@ -148,181 +53,19 @@
 		#size-cells = <2>;
 		ranges = <0x02000000 0 0x10000000 0 0x10000000 0 0x2000
 				0x01000000 0 0x20000000 0 0x20000000 0 0x2000>;
-		pci@1f,0 {
-			compatible = "pci-generic";
-			reg = <0xf800 0 0 0 0>;
-			emul@1f,0 {
-				compatible = "sandbox,swap-case";
-			};
-		};
 	};
 
-	pinctrl {
-		compatible = "sandbox,pinctrl";
-
-		pinctrl_i2c0: i2c0 {
-			groups = "i2c";
-			function = "i2c";
-			bias-pull-up;
-		};
-
-		pinctrl_serial0: uart0 {
-			groups = "serial_a";
-			function = "serial";
-		};
-	};
-
-	reset@1 {
-		compatible = "sandbox,reset";
-	};
-
-	spi@0 {
+	spi: spi@0 {
+		u-boot,dm-pre-reloc;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		reg = <0 0 0 0>;
 		compatible = "sandbox,spi";
 		cs-gpios = <0>, <&gpio_a 0>;
-		firmware_storage_spi: flash@0 {
-			reg = <0>;
-			compatible = "spansion,m25p16", "sandbox,spi-flash";
-			spi-max-frequency = <40000000>;
-			sandbox,filename = "spi.bin";
-		};
-	};
-
-	spl-test {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,spl-test";
-		boolval;
-		intval = <1>;
-		intarray = <2 3 4>;
-		byteval = [05];
-		bytearray = [06];
-		longbytearray = [09 0a 0b 0c 0d 0e 0f 10 11];
-		stringval = "message";
-		stringarray = "multi-word", "message";
-	};
-
-	spl-test2 {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,spl-test";
-		intval = <3>;
-		intarray = <5>;
-		byteval = [08];
-		bytearray = [01 23 34];
-		longbytearray = [09 0a 0b 0c];
-		stringval = "message2";
-		stringarray = "another", "multi-word", "message";
-	};
-
-	spl-test3 {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,spl-test";
-		stringarray = "one";
-	};
-
-	spl-test4 {
-		u-boot,dm-pre-reloc;
-		compatible = "sandbox,spl-test.2";
 	};
 
-	square {
-		compatible = "demo-shape";
-		colour = "blue";
-		sides = <4>;
-	};
-
-	timer {
-		compatible = "sandbox,timer";
-		clock-frequency = <1000000>;
-	};
-
-	tpm {
-		compatible = "google,sandbox-tpm";
-	};
-
-	tpm2 {
-		compatible = "sandbox,tpm2";
-	};
-
-	triangle {
-		compatible = "demo-shape";
-		colour = "cyan";
-		sides = <3>;
-		character = <83>;
-		light-gpios = <&gpio_a 2>, <&gpio_b 6 0>;
-	};
-
-	/* Needs to be available prior to relocation */
-	uart0: serial {
-		compatible = "sandbox,serial";
-		sandbox,text-colour = "cyan";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_serial0>;
-	};
-
-	usb@0 {
-		compatible = "sandbox,usb";
-		status = "disabled";
-		hub {
-			compatible = "sandbox,usb-hub";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			flash-stick {
-				reg = <0>;
-				compatible = "sandbox,usb-flash";
-			};
-		};
-	};
-
-	usb@1 {
-		compatible = "sandbox,usb";
-		hub {
-			compatible = "usb-hub";
-			usb,device-class = <USB_CLASS_HUB>;
-			hub-emul {
-				compatible = "sandbox,usb-hub";
-				#address-cells = <1>;
-				#size-cells = <0>;
-				flash-stick {
-					reg = <0>;
-					compatible = "sandbox,usb-flash";
-					sandbox,filepath = "flash.bin";
-				};
-			};
-		};
-	};
-
-	usb@2 {
-		compatible = "sandbox,usb";
-		status = "disabled";
-	};
-
-	spmi: spmi@0 {
-		compatible = "sandbox,spmi";
-		#address-cells = <0x1>;
-		#size-cells = <0x1>;
-		pm8916@0 {
-			compatible = "qcom,spmi-pmic";
-			reg = <0x0 0x1>;
-			#address-cells = <0x1>;
-			#size-cells = <0x1>;
-
-			spmi_gpios: gpios@c000 {
-				compatible = "qcom,pm8916-gpio";
-				reg = <0xc000 0x400>;
-				gpio-controller;
-				gpio-count = <4>;
-				#gpio-cells = <2>;
-				gpio-bank-name="spmi";
-			};
-		};
-	};
-
-	sandbox_tee {
-		compatible = "sandbox,tee";
-	};
 };
 
+#include "sandbox.dtsi"
 #include "cros-ec-keyboard.dtsi"
 #include "sandbox_pmic.dtsi"

arch/sandbox/dts/test.dts

@@ -446,6 +446,14 @@
 				compatible = "sandbox,swap-case";
 			};
 		};
+		pci@1,0 {
+			compatible = "pci-generic";
+			reg = <0x0800 0 0 0 0>;
+			emul@0,0 {
+				compatible = "sandbox,swap-case";
+				use-ea;
+			};
+		};
 		pci@1f,0 {
 			compatible = "pci-generic";
 			reg = <0xf800 0 0 0 0>;

arch/sandbox/include/asm/test.h

@@ -19,6 +19,7 @@
 #define PCI_CAP_ID_PM_OFFSET		0x50
 #define PCI_CAP_ID_EXP_OFFSET		0x60
 #define PCI_CAP_ID_MSIX_OFFSET		0x70
+#define PCI_CAP_ID_EA_OFFSET		0x80
 
 #define PCI_EXT_CAP_ID_ERR_OFFSET	0x100
 #define PCI_EXT_CAP_ID_VC_OFFSET	0x200
@@ -30,6 +31,18 @@
 
 #define SANDBOX_CLK_RATE		32768
 
+/* Macros used to test PCI EA capability structure */
+#define PCI_CAP_EA_BASE_LO0		0x00100000
+#define PCI_CAP_EA_BASE_LO1		0x00110000
+#define PCI_CAP_EA_BASE_LO2		0x00120000
+#define PCI_CAP_EA_BASE_LO4		0x00140000
+#define PCI_CAP_EA_BASE_HI2		0x00020000ULL
+#define PCI_CAP_EA_BASE_HI4		0x00040000ULL
+#define PCI_CAP_EA_SIZE_LO		0x0000ffff
+#define PCI_CAP_EA_SIZE_HI		0x00000010ULL
+#define PCI_EA_BAR2_MAGIC		0x72727272
+#define PCI_EA_BAR4_MAGIC		0x74747474
+
 /* System controller driver data */
 enum {
 	SYSCON0		= 32,

arch/x86/include/asm/coreboot_tables.h

@@ -8,6 +8,8 @@
 #ifndef _COREBOOT_TABLES_H
 #define _COREBOOT_TABLES_H
 
+struct memory_area;
+
 struct cbuint64 {
 	u32 lo;
 	u32 hi;

board/sandbox/README.sandbox

@@ -435,6 +435,27 @@ board_init_f() and board_init_r().
 This approach can be used on normal boards as well as sandbox.
 
 
+SDL_CONFIG
+----------
+
+If sdl-config is on a different path from the default, set the SDL_CONFIG
+environment variable to the correct pathname before building U-Boot.
+
+
+Using valgrind / memcheck
+-------------------------
+
+It is possible to run U-Boot under valgrind to check memory allocations:
+
+   valgrind u-boot
+
+If you are running sandbox SPL or TPL, then valgrind will not by default
+notice when U-Boot jumps from TPL to SPL, or from SPL to U-Boot proper. To
+fix this, use:
+
+   valgrind --trace-children=yes u-boot
+
+
 Testing
 -------
 

cmd/trace.c

@@ -30,8 +30,7 @@ static int get_args(int argc, char * const argv[], char **buff,
 
 static int create_func_list(int argc, char * const argv[])
 {
-	size_t buff_size, avail, buff_ptr, used;
-	unsigned int needed;
+	size_t buff_size, avail, buff_ptr, needed, used;
 	char *buff;
 	int err;
 
@@ -41,7 +40,7 @@ static int create_func_list(int argc, char * const argv[])
 	avail = buff_size - buff_ptr;
 	err = trace_list_functions(buff + buff_ptr, avail, &needed);
 	if (err)
-		printf("Error: truncated (%#x bytes needed)\n", needed);
+		printf("Error: truncated (%#zx bytes needed)\n", needed);
 	used = min(avail, (size_t)needed);
 	printf("Function trace dumped to %08lx, size %#zx\n",
 	       (ulong)map_to_sysmem(buff + buff_ptr), used);
@@ -54,8 +53,7 @@ static int create_func_list(int argc, char * const argv[])
 
 static int create_call_list(int argc, char * const argv[])
 {
-	size_t buff_size, avail, buff_ptr, used;
-	unsigned int needed;
+	size_t buff_size, avail, buff_ptr, needed, used;
 	char *buff;
 	int err;
 
@@ -65,7 +63,7 @@ static int create_call_list(int argc, char * const argv[])
 	avail = buff_size - buff_ptr;
 	err = trace_list_calls(buff + buff_ptr, avail, &needed);
 	if (err)
-		printf("Error: truncated (%#x bytes needed)\n", needed);
+		printf("Error: truncated (%#zx bytes needed)\n", needed);
 	used = min(avail, (size_t)needed);
 	printf("Call list dumped to %08lx, size %#zx\n",
 	       (ulong)map_to_sysmem(buff + buff_ptr), used);

common/Kconfig

@@ -69,6 +69,13 @@ config SPL_BOOTSTAGE_RECORD_COUNT
 	  This is the size of the bootstage record list and is the maximum
 	  number of bootstage records that can be recorded.
 
+config TPL_BOOTSTAGE_RECORD_COUNT
+	int "Number of boot stage records to store for TPL"
+	default 5
+	help
+	  This is the size of the bootstage record list and is the maximum
+	  number of bootstage records that can be recorded.
+
 config BOOTSTAGE_FDT
 	bool "Store boot timing information in the OS device tree"
 	depends on BOOTSTAGE

common/board_f.c

@@ -442,8 +442,8 @@ static int reserve_trace(void)
 #ifdef CONFIG_TRACE
 	gd->relocaddr -= CONFIG_TRACE_BUFFER_SIZE;
 	gd->trace_buff = map_sysmem(gd->relocaddr, CONFIG_TRACE_BUFFER_SIZE);
-	debug("Reserving %dk for trace data at: %08lx\n",
-	      CONFIG_TRACE_BUFFER_SIZE >> 10, gd->relocaddr);
+	debug("Reserving %luk for trace data at: %08lx\n",
+	      (unsigned long)CONFIG_TRACE_BUFFER_SIZE >> 10, gd->relocaddr);
 #endif
 
 	return 0;
@@ -839,7 +839,7 @@ static const init_fnc_t init_sequence_f[] = {
 #ifdef CONFIG_OF_CONTROL
 	fdtdec_setup,
 #endif
-#ifdef CONFIG_TRACE
+#ifdef CONFIG_TRACE_EARLY
 	trace_early_init,
 #endif
 	initf_malloc,

doc/README.SPL

@@ -66,6 +66,22 @@ CONFIG_SPL_SPI_LOAD (drivers/mtd/spi/spi_spl_load.o)
 CONFIG_SPL_RAM_DEVICE (common/spl/spl.c)
 CONFIG_SPL_WATCHDOG_SUPPORT (drivers/watchdog/libwatchdog.o)
 
+Device tree
+-----------
+The U-Boot device tree is filtered by the fdtgrep tools during the build
+process to generate a much smaller device tree used in SPL (spl/u-boot-spl.dtb)
+with:
+- the mandatory nodes (/alias, /chosen, /config)
+- the nodes with one pre-relocation property:
+  'u-boot,dm-pre-reloc' or 'u-boot,dm-spl'
+
+ftgrep is also used to remove:
+- the properties defined in CONFIG_OF_SPL_REMOVE_PROPS
+- all the pre-relocation properties
+  ('u-boot,dm-pre-reloc', 'u-boot,dm-spl' and 'u-boot,dm-tpl')
+
+All the nodes remaining in the SPL devicetree are bound
+(see driver-model/README.txt).
 
 Debugging
 ---------

doc/README.TPL

@@ -34,6 +34,10 @@ determine which SPL options to choose based on whether CONFIG_TPL_BUILD
 is set. Source files can be compiled for TPL with options choosed in the
 board config file.
 
+TPL use a small device tree (u-boot-tpl.dtb), containing only the nodes with
+the pre-relocation properties: 'u-boot,dm-pre-reloc' and 'u-boot,dm-tpl'
+(see README.SPL for details).
+
 For example:
 
 spl/Makefile:

doc/driver-model/README.txt

@@ -849,6 +849,10 @@ in the device tree node. For U-Boot proper you can use 'u-boot,dm-pre-proper'
 which means that it will be processed (and a driver bound) in U-Boot proper
 prior to relocation, but will not be available in SPL or TPL.
 
+To reduce the size of SPL and TPL, only the nodes with pre-relocation properties
+('u-boot,dm-pre-reloc', 'u-boot,dm-spl' or 'u-boot,dm-tpl') are keept in their
+device trees (see README.SPL for details); the remaining nodes are always bound.
+
 Then post relocation we throw that away and re-init driver model again.
 For drivers which require some sort of continuity between pre- and
 post-relocation devices, we can provide access to the pre-relocation

drivers/block/Kconfig

@@ -56,6 +56,13 @@ config SPL_BLOCK_CACHE
 	help
 	  This option enables the disk-block cache in SPL
 
+config TPL_BLOCK_CACHE
+	bool "Use block device cache in TPL"
+	depends on TPL_BLK
+	default n
+	help
+	  This option enables the disk-block cache in TPL
+
 config IDE
 	bool "Support IDE controllers"
 	select HAVE_BLOCK_DEVICE

drivers/core/util.c

@@ -42,6 +42,8 @@ bool dm_ofnode_pre_reloc(ofnode node)
 #else
 	if (ofnode_read_bool(node, "u-boot,dm-pre-reloc"))
 		return true;
+	if (ofnode_read_bool(node, "u-boot,dm-pre-proper"))
+		return true;
 
 	/*
 	 * In regular builds individual spl and tpl handling both

drivers/misc/swap_case.c

@@ -61,11 +61,63 @@ static int sandbox_swap_case_get_devfn(struct udevice *dev)
 	return plat->devfn;
 }
 
+static int sandbox_swap_case_use_ea(struct udevice *dev)
+{
+	return !!ofnode_get_property(dev->node, "use-ea", NULL);
+}
+
+/* Please keep these macros in sync with ea_regs below */
+#define PCI_CAP_ID_EA_SIZE		(sizeof(ea_regs) + 4)
+#define PCI_CAP_ID_EA_ENTRY_CNT		4
+/* Hardcoded EA structure, excluding 1st DW. */
+static const u32 ea_regs[] = {
+	/* BEI=0, ES=2, BAR0 32b Base + 32b MaxOffset, I/O space */
+	(2 << 8) | 2,
+	PCI_CAP_EA_BASE_LO0,
+	0,
+	/* BEI=1, ES=2, BAR1 32b Base + 32b MaxOffset */
+	(1 << 4) | 2,
+	PCI_CAP_EA_BASE_LO1,
+	MEM_TEXT_SIZE - 1,
+	/* BEI=2, ES=3, BAR2 64b Base + 32b MaxOffset */
+	(2 << 4) | 3,
+	PCI_CAP_EA_BASE_LO2 | PCI_EA_IS_64,
+	PCI_CAP_EA_SIZE_LO,
+	PCI_CAP_EA_BASE_HI2,
+	/* BEI=4, ES=4, BAR4 64b Base + 64b MaxOffset */
+	(4 << 4) | 4,
+	PCI_CAP_EA_BASE_LO4 | PCI_EA_IS_64,
+	PCI_CAP_EA_SIZE_LO | PCI_EA_IS_64,
+	PCI_CAP_EA_BASE_HI4,
+	PCI_CAP_EA_SIZE_HI,
+};
+
+static int sandbox_swap_case_read_ea(struct udevice *emul, uint offset,
+				     ulong *valuep, enum pci_size_t size)
+{
+	u32 reg;
+
+	offset = offset - PCI_CAP_ID_EA_OFFSET - 4;
+	reg = ea_regs[offset >> 2];
+	reg >>= (offset % 4) * 8;
+
+	*valuep = reg;
+	return 0;
+}
+
 static int sandbox_swap_case_read_config(struct udevice *emul, uint offset,
 					 ulong *valuep, enum pci_size_t size)
 {
 	struct swap_case_platdata *plat = dev_get_platdata(emul);
 
+	/*
+	 * The content of the EA capability structure is handled elsewhere to
+	 * keep the switch/case below sane
+	 */
+	if (offset > PCI_CAP_ID_EA_OFFSET + PCI_CAP_LIST_NEXT &&
+	    offset < PCI_CAP_ID_EA_OFFSET + PCI_CAP_ID_EA_SIZE)
+		return sandbox_swap_case_read_ea(emul, offset, valuep, size);
+
 	switch (offset) {
 	case PCI_COMMAND:
 		*valuep = plat->command;
@@ -134,9 +186,21 @@ static int sandbox_swap_case_read_config(struct udevice *emul, uint offset,
 		*valuep = PCI_CAP_ID_MSIX_OFFSET;
 		break;
 	case PCI_CAP_ID_MSIX_OFFSET:
+		if (sandbox_swap_case_use_ea(emul))
+			*valuep = (PCI_CAP_ID_EA_OFFSET << 8) | PCI_CAP_ID_MSIX;
+		else
 			*valuep = PCI_CAP_ID_MSIX;
 		break;
 	case PCI_CAP_ID_MSIX_OFFSET + PCI_CAP_LIST_NEXT:
+		if (sandbox_swap_case_use_ea(emul))
+			*valuep = PCI_CAP_ID_EA_OFFSET;
+		else
+			*valuep = 0;
+		break;
+	case PCI_CAP_ID_EA_OFFSET:
+		*valuep = (PCI_CAP_ID_EA_ENTRY_CNT << 16) | PCI_CAP_ID_EA;
+		break;
+	case PCI_CAP_ID_EA_OFFSET + PCI_CAP_LIST_NEXT:
 		*valuep = 0;
 		break;
 	case PCI_EXT_CAP_ID_ERR_OFFSET:
@@ -257,6 +321,9 @@ int sandbox_swap_case_write_io(struct udevice *dev, unsigned int addr,
 	return 0;
 }
 
+static int pci_ea_bar2_magic = PCI_EA_BAR2_MAGIC;
+static int pci_ea_bar4_magic = PCI_EA_BAR4_MAGIC;
+
 static int sandbox_swap_case_map_physmem(struct udevice *dev,
 		phys_addr_t addr, unsigned long *lenp, void **ptrp)
 {
@@ -265,9 +332,42 @@ static int sandbox_swap_case_map_physmem(struct udevice *dev,
 	int barnum;
 	int ret;
 
+	if (sandbox_swap_case_use_ea(dev)) {
+		/*
+		 * only support mapping base address in EA test for now, we
+		 * don't handle mapping an offset inside a BAR.  Seems good
+		 * enough for the current test.
+		 */
+		switch (addr) {
+		case (phys_addr_t)PCI_CAP_EA_BASE_LO0:
+			*ptrp = &priv->op;
+			*lenp = 4;
+			break;
+		case (phys_addr_t)PCI_CAP_EA_BASE_LO1:
+			*ptrp = priv->mem_text;
+			*lenp = barinfo[1].size - 1;
+			break;
+		case (phys_addr_t)((PCI_CAP_EA_BASE_HI2 << 32) |
+				   PCI_CAP_EA_BASE_LO2):
+			*ptrp = &pci_ea_bar2_magic;
+			*lenp = PCI_CAP_EA_SIZE_LO;
+			break;
+		case (phys_addr_t)((PCI_CAP_EA_BASE_HI4 << 32) |
+				   PCI_CAP_EA_BASE_LO4):
+			*ptrp = &pci_ea_bar4_magic;
+			*lenp = (PCI_CAP_EA_SIZE_HI << 32) |
+				PCI_CAP_EA_SIZE_LO;
+			break;
+		default:
+			return -ENOENT;
+		}
+		return 0;
+	}
+
 	ret = sandbox_swap_case_find_bar(dev, addr, &barnum, &offset);
 	if (ret)
 		return ret;
+
 	if (barnum == 1) {
 		*ptrp = priv->mem_text + offset;
 		avail = barinfo[1].size - offset;

drivers/pci/pci-uclass.c

@@ -1341,10 +1341,56 @@ pci_addr_t dm_pci_phys_to_bus(struct udevice *dev, phys_addr_t phys_addr,
 	return bus_addr;
 }
 
+static void *dm_pci_map_ea_bar(struct udevice *dev, int bar, int flags,
+			       int ea_off)
+{
+	int ea_cnt, i, entry_size;
+	int bar_id = (bar - PCI_BASE_ADDRESS_0) >> 2;
+	u32 ea_entry;
+	phys_addr_t addr;
+
+	/* EA capability structure header */
+	dm_pci_read_config32(dev, ea_off, &ea_entry);
+	ea_cnt = (ea_entry >> 16) & PCI_EA_NUM_ENT_MASK;
+	ea_off += PCI_EA_FIRST_ENT;
+
+	for (i = 0; i < ea_cnt; i++, ea_off += entry_size) {
+		/* Entry header */
+		dm_pci_read_config32(dev, ea_off, &ea_entry);
+		entry_size = ((ea_entry & PCI_EA_ES) + 1) << 2;
+
+		if (((ea_entry & PCI_EA_BEI) >> 4) != bar_id)
+			continue;
+
+		/* Base address, 1st DW */
+		dm_pci_read_config32(dev, ea_off + 4, &ea_entry);
+		addr = ea_entry & PCI_EA_FIELD_MASK;
+		if (ea_entry & PCI_EA_IS_64) {
+			/* Base address, 2nd DW, skip over 4B MaxOffset */
+			dm_pci_read_config32(dev, ea_off + 12, &ea_entry);
+			addr |= ((u64)ea_entry) << 32;
+		}
+
+		/* size ignored for now */
+		return map_physmem(addr, flags, 0);
+	}
+
+	return 0;
+}
+
 void *dm_pci_map_bar(struct udevice *dev, int bar, int flags)
 {
 	pci_addr_t pci_bus_addr;
 	u32 bar_response;
+	int ea_off;
+
+	/*
+	 * if the function supports Enhanced Allocation use that instead of
+	 * BARs
+	 */
+	ea_off = dm_pci_find_capability(dev, PCI_CAP_ID_EA);
+	if (ea_off)
+		return dm_pci_map_ea_bar(dev, bar, flags, ea_off);
 
 	/* read BAR address */
 	dm_pci_read_config32(dev, bar, &bar_response);
@@ -1448,6 +1494,30 @@ int dm_pci_find_ext_capability(struct udevice *dev, int cap)
 	return dm_pci_find_next_ext_capability(dev, 0, cap);
 }
 
+int dm_pci_flr(struct udevice *dev)
+{
+	int pcie_off;
+	u32 cap;
+
+	/* look for PCI Express Capability */
+	pcie_off = dm_pci_find_capability(dev, PCI_CAP_ID_EXP);
+	if (!pcie_off)
+		return -ENOENT;
+
+	/* check FLR capability */
+	dm_pci_read_config32(dev, pcie_off + PCI_EXP_DEVCAP, &cap);
+	if (!(cap & PCI_EXP_DEVCAP_FLR))
+		return -ENOENT;
+
+	dm_pci_clrset_config16(dev, pcie_off + PCI_EXP_DEVCTL, 0,
+			       PCI_EXP_DEVCTL_BCR_FLR);
+
+	/* wait 100ms, per PCI spec */
+	mdelay(100);
+
+	return 0;
+}
+
 UCLASS_DRIVER(pci) = {
 	.id		= UCLASS_PCI,
 	.name		= "pci",

drivers/serial/serial_lpuart.c

 // SPDX-License-Identifier: GPL-2.0+
 /*
+ * Copyright 2019 NXP
  * Copyright 2013 Freescale Semiconductor, Inc.
  */
 
@@ -502,6 +503,9 @@ static int lpuart_serial_ofdata_to_platdata(struct udevice *dev)
 	plat->reg = (void *)addr;
 	plat->flags = dev_get_driver_data(dev);
 
+	if (fdtdec_get_bool(blob, node, "little-endian"))
+		plat->flags &= ~LPUART_FLAG_REGMAP_ENDIAN_BIG;
+
 	if (!fdt_node_check_compatible(blob, node, "fsl,ls1021a-lpuart"))
 		plat->devtype = DEV_LS1021A;
 	else if (!fdt_node_check_compatible(blob, node, "fsl,imx7ulp-lpuart"))

drivers/spi/Kconfig

@@ -226,7 +226,7 @@ config SANDBOX_SPI
 		cs-gpios = <0>, <&gpio_a 0>;
 		flash@0 {
 			reg = <0>;
-			compatible = "spansion,m25p16", "sandbox,spi-flash";
+			compatible = "spansion,m25p16", "jedec,spi-nor";
 			spi-max-frequency = <40000000>;
 			sandbox,filename = "spi.bin";
 		};

drivers/spi/spi-mem.c

@@ -201,7 +201,6 @@ int spi_mem_exec_op(struct spi_slave *slave, const struct spi_mem_op *op)
 	unsigned int pos = 0;
 	const u8 *tx_buf = NULL;
 	u8 *rx_buf = NULL;
-	u8 *op_buf;
 	int op_len;
 	u32 flag;
 	int ret;
@@ -338,7 +337,17 @@ int spi_mem_exec_op(struct spi_slave *slave, const struct spi_mem_op *op)
 	}
 
 	op_len = sizeof(op->cmd.opcode) + op->addr.nbytes + op->dummy.nbytes;
-	op_buf = calloc(1, op_len);
+
+	/*
+	 * Avoid using malloc() here so that we can use this code in SPL where
+	 * simple malloc may be used. That implementation does not allow free()
+	 * so repeated calls to this code can exhaust the space.
+	 *
+	 * The value of op_len is small, since it does not include the actual
+	 * data being sent, only the op-code and address. In fact, it should be
+	 * possible to just use a small fixed value here instead of op_len.
+	 */
+	u8 op_buf[op_len];
 
 	op_buf[pos++] = op->cmd.opcode;
 
@@ -382,8 +391,6 @@ int spi_mem_exec_op(struct spi_slave *slave, const struct spi_mem_op *op)
 		debug("%02x ", tx_buf ? tx_buf[i] : rx_buf[i]);
 	debug("[ret %d]\n", ret);
 
-	free(op_buf);
-
 	if (ret < 0)
 		return ret;
 #endif /* __UBOOT__ */

drivers/sysreset/sysreset_sandbox.c

@@ -66,6 +66,7 @@ static int sandbox_sysreset_request(struct udevice *dev, enum sysreset_t type)
 	case SYSRESET_POWER_OFF:
 		if (!state->sysreset_allowed[type])
 			return -EACCES;
+		sandbox_exit();
 	default:
 		return -ENOSYS;
 	}

drivers/video/video-uclass.c

@@ -291,7 +291,9 @@ static int video_post_bind(struct udevice *dev)
 		return 0;
 	size = alloc_fb(dev, &addr);
 	if (addr < gd->video_bottom) {
-		/* Device tree node may need the 'u-boot,dm-pre-reloc' tag */
+		/* Device tree node may need the 'u-boot,dm-pre-reloc' or
+		 * 'u-boot,dm-pre-proper' tag
+		 */
 		printf("Video device '%s' cannot allocate frame buffer memory -ensure the device is set up before relocation\n",
 		       dev->name);
 		return -ENOSPC;

include/configs/sandbox.h

@@ -63,7 +63,11 @@
 	func(HOST, host, 1) \
 	func(HOST, host, 0)
 
+#ifdef __ASSEMBLY__
+#define BOOTENV
+#else
 #include <config_distro_bootcmd.h>
+#endif
 
 #define CONFIG_KEEP_SERVERADDR
 #define CONFIG_UDP_CHECKSUM

include/dm/ofnode.h

@@ -676,12 +676,14 @@ int ofnode_read_simple_size_cells(ofnode node);
  * After relocation and jumping into the real U-Boot binary it is possible to
  * determine if a node was bound in one of SPL/TPL stages.
  *
- * There are 3 settings currently in use
- * -
+ * There are 4 settings currently in use
+ * - u-boot,dm-pre-proper: U-Boot proper pre-relocation only
  * - u-boot,dm-pre-reloc: legacy and indicates any of TPL or SPL
  *   Existing platforms only use it to indicate nodes needed in
  *   SPL. Should probably be replaced by u-boot,dm-spl for
  *   new platforms.
+ * - u-boot,dm-spl: SPL and U-Boot pre-relocation
+ * - u-boot,dm-tpl: TPL and U-Boot pre-relocation
  *
  * @node: node to check
  * @return true if node is needed in SPL/TL, false otherwise

include/dm/util.h

@@ -52,12 +52,14 @@ static inline void dm_dump_devres(void)
  * it is possible to determine if a node was bound in one of
  * SPL/TPL stages.
  *
- * There are 3 settings currently in use
- * -
+ * There are 4 settings currently in use
+ * - u-boot,dm-pre-proper: U-Boot proper pre-relocation only
  * - u-boot,dm-pre-reloc: legacy and indicates any of TPL or SPL
  *   Existing platforms only use it to indicate nodes needed in
  *   SPL. Should probably be replaced by u-boot,dm-spl for
  *   existing platforms.
+ * - u-boot,dm-spl: SPL and U-Boot pre-relocation
+ * - u-boot,dm-tpl: TPL and U-Boot pre-relocation
  * @node: of node
  *
  * Returns true if node is needed in SPL/TL, false otherwise.

include/pci.h

@@ -455,6 +455,23 @@
 #define PCI_EXT_CAP_ID_PTM	0x1F	/* Precision Time Measurement */
 #define PCI_EXT_CAP_ID_MAX	PCI_EXT_CAP_ID_PTM
 
+/* Enhanced Allocation Registers */
+#define PCI_EA_NUM_ENT		2	/* Number of Capability Entries */
+#define  PCI_EA_NUM_ENT_MASK	0x3f	/* Num Entries Mask */
+#define PCI_EA_FIRST_ENT	4	/* First EA Entry in List */
+#define  PCI_EA_ES		0x00000007 /* Entry Size */
+#define  PCI_EA_BEI		0x000000f0 /* BAR Equivalent Indicator */
+/* Base, MaxOffset registers */
+/* bit 0 is reserved */
+#define  PCI_EA_IS_64		0x00000002	/* 64-bit field flag */
+#define  PCI_EA_FIELD_MASK	0xfffffffc	/* For Base & Max Offset */
+
+/* PCI Express capabilities */
+#define PCI_EXP_DEVCAP		4	/* Device capabilities */
+#define  PCI_EXP_DEVCAP_FLR     0x10000000 /* Function Level Reset */
+#define PCI_EXP_DEVCTL		8	/* Device Control */
+#define  PCI_EXP_DEVCTL_BCR_FLR 0x8000  /* Bridge Configuration Retry / FLR */
+
 /* Include the ID list */
 
 #include <pci_ids.h>
@@ -1309,12 +1326,16 @@ pci_addr_t dm_pci_phys_to_bus(struct udevice *dev, phys_addr_t addr,
  * dm_pci_map_bar() - get a virtual address associated with a BAR region
  *
  * Looks up a base address register and finds the physical memory address
- * that corresponds to it
+ * that corresponds to it.
+ * Can be used for 32b BARs 0-5 on type 0 functions and for 32b BARs 0-1 on
+ * type 1 functions.
+ * Can also be used on type 0 functions that support Enhanced Allocation for
+ * 32b/64b BARs.  Note that duplicate BEI entries are not supported.
  *
  * @dev:	Device to check
- * @bar:	Bar number to read (numbered from 0)
+ * @bar:	Bar register offset (PCI_BASE_ADDRESS_...)
  * @flags:	Flags for the region type (PCI_REGION_...)
- * @return: pointer to the virtual address to use
+ * @return: pointer to the virtual address to use or 0 on error
  */
 void *dm_pci_map_bar(struct udevice *dev, int bar, int flags);
 
@@ -1411,6 +1432,14 @@ int dm_pci_find_next_ext_capability(struct udevice *dev, int start, int cap);
  */
 int dm_pci_find_ext_capability(struct udevice *dev, int cap);
 
+/**
+ * dm_pci_flr() - Perform FLR if the device suppoorts it
+ *
+ * @dev:	PCI device to reset
+ * @return:	0 if OK, -ENOENT if FLR is not supported by dev
+ */
+int dm_pci_flr(struct udevice *dev);
+
 #define dm_pci_virt_to_bus(dev, addr, flags) \
 	dm_pci_phys_to_bus(dev, (virt_to_phys(addr)), (flags))
 #define dm_pci_bus_to_virt(dev, addr, flags, len, map_flags) \

include/trace.h

@@ -39,7 +39,7 @@ struct trace_output_func {
 /* A header at the start of the trace output buffer */
 struct trace_output_hdr {
 	enum trace_chunk_type type;	/* Record type */
-	uint32_t rec_count;		/* Number of records */
+	size_t rec_count;		/* Number of records */
 };
 
 /* Print statistics about traced function calls */
@@ -57,7 +57,7 @@ void trace_print_stats(void);
  * @param needed	Returns number of bytes used / needed
  * @return 0 if ok, -1 on error (buffer exhausted)
  */
-int trace_list_functions(void *buff, int buff_size, unsigned *needed);
+int trace_list_functions(void *buff, size_t buff_size, size_t *needed);
 
 /* Flags for ftrace_record */
 enum ftrace_flags {
@@ -77,7 +77,7 @@ struct trace_call {
 	uint32_t flags;		/* Flags and timestamp */
 };
 
-int trace_list_calls(void *buff, int buff_size, unsigned int *needed);
+int trace_list_calls(void *buff, size_t buff_size, size_t *needed);
 
 /**
  * Turn function tracing on and off

lib/Kconfig

@@ -192,6 +192,13 @@ config TRACE_BUFFER_SIZE
 	  the size is too small then 'trace stats' will show a message saying
 	  how many records were dropped due to buffer overflow.
 
+config TRACE_CALL_DEPTH_LIMIT
+	int "Trace call depth limit"
+	depends on TRACE
+	default 15
+	help
+	  Sets the maximum call depth up to which function calls are recorded.
+
 config TRACE_EARLY
 	bool "Enable tracing before relocation"
 	depends on TRACE
@@ -209,6 +216,14 @@ config TRACE_EARLY_SIZE
 	  Sets the size of the early trace buffer in bytes. This is used to hold
 	  tracing information before relocation.
 
+config TRACE_EARLY_CALL_DEPTH_LIMIT
+	int "Early trace call depth limit"
+	depends on TRACE_EARLY
+	default 200
+	help
+	  Sets the maximum call depth up to which function calls are recorded
+	  during early tracing.
+
 config TRACE_EARLY_ADDR
 	hex "Address of early trace buffer in U-Boot"
 	depends on TRACE_EARLY

lib/fdtdec_test.c

@@ -138,6 +138,7 @@ static int run_test(const char *aliases, const char *nodes, const char *expect)
 	}
 
 	printf("pass\n");
+	free(blob);
 	return 0;
 }
 
@@ -292,6 +293,7 @@ static int check_carveout(void)
 	CHECKVAL(make_fdt_carveout(fdt, FDT_SIZE, 2, 2), 0);
 	CHECKOK(check_fdt_carveout(fdt, 2, 2));
 
+	free(fdt);
 	return 0;
 }
 

lib/hang.c

@@ -9,6 +9,7 @@
 
 #include <common.h>
 #include <bootstage.h>
+#include <os.h>
 
 /**
  * hang - stop processing by staying in an endless loop
@@ -26,6 +27,8 @@ void hang(void)
 	puts("### ERROR ### Please RESET the board ###\n");
 #endif
 	bootstage_error(BOOTSTAGE_ID_NEED_RESET);
+	if (IS_ENABLED(CONFIG_SANDBOX))
+		os_exit(1);
 	for (;;)
 		;
 }

lib/trace.c

@@ -56,6 +56,49 @@ static inline uintptr_t __attribute__((no_instrument_function))
 	return offset / FUNC_SITE_SIZE;
 }
 
+#ifdef CONFIG_EFI_LOADER
+
+/**
+ * trace_gd - the value of the gd register
+ */
+static volatile void *trace_gd;
+
+/**
+ * trace_save_gd() - save the value of the gd register
+ */
+static void __attribute__((no_instrument_function)) trace_save_gd(void)
+{
+	trace_gd = gd;
+}
+
+/**
+ * trace_swap_gd() - swap between U-Boot and application gd register value
+ *
+ * An UEFI application may change the value of the register that gd lives in.
+ * But some of our functions like get_ticks() access this register. So we
+ * have to set the gd register to the U-Boot value when entering a trace
+ * point and set it back to the application value when exiting the trace point.
+ */
+static void __attribute__((no_instrument_function)) trace_swap_gd(void)
+{
+	volatile void *temp_gd = trace_gd;
+
+	trace_gd = gd;
+	gd = temp_gd;
+}
+
+#else
+
+static void __attribute__((no_instrument_function)) trace_save_gd(void)
+{
+}
+
+static void __attribute__((no_instrument_function)) trace_swap_gd(void)
+{
+}
+
+#endif
+
 static void __attribute__((no_instrument_function)) add_ftrace(void *func_ptr,
 				void *caller, ulong flags)
 {
@@ -100,6 +143,7 @@ void __attribute__((no_instrument_function)) __cyg_profile_func_enter(
 	if (trace_enabled) {
 		int func;
 
+		trace_swap_gd();
 		add_ftrace(func_ptr, caller, FUNCF_ENTRY);
 		func = func_ptr_to_num(func_ptr);
 		if (func < hdr->func_count) {
@@ -111,6 +155,7 @@ void __attribute__((no_instrument_function)) __cyg_profile_func_enter(
 		hdr->depth++;
 		if (hdr->depth > hdr->depth_limit)
 			hdr->max_depth = hdr->depth;
+		trace_swap_gd();
 	}
 }
 
@@ -126,8 +171,10 @@ void __attribute__((no_instrument_function)) __cyg_profile_func_exit(
 		void *func_ptr, void *caller)
 {
 	if (trace_enabled) {
+		trace_swap_gd();
 		add_ftrace(func_ptr, caller, FUNCF_EXIT);
 		hdr->depth--;
+		trace_swap_gd();
 	}
 }
 
@@ -143,12 +190,12 @@ void __attribute__((no_instrument_function)) __cyg_profile_func_exit(
  *			greater than buff_size if we ran out of space.
  * @return 0 if ok, -1 if space was exhausted
  */
-int trace_list_functions(void *buff, int buff_size, unsigned int *needed)
+int trace_list_functions(void *buff, size_t buff_size, size_t *needed)
 {
 	struct trace_output_hdr *output_hdr = NULL;
 	void *end, *ptr = buff;
-	int func;
-	int upto;
+	size_t func;
+	size_t upto;
 
 	end = buff ? buff + buff_size : NULL;
 
@@ -159,7 +206,7 @@ int trace_list_functions(void *buff, int buff_size, unsigned int *needed)
 
 	/* Add information about each function */
 	for (func = upto = 0; func < hdr->func_count; func++) {
-		int calls = hdr->call_accum[func];
+		size_t calls = hdr->call_accum[func];
 
 		if (!calls)
 			continue;
@@ -188,12 +235,12 @@ int trace_list_functions(void *buff, int buff_size, unsigned int *needed)
 	return 0;
 }
 
-int trace_list_calls(void *buff, int buff_size, unsigned *needed)
+int trace_list_calls(void *buff, size_t buff_size, size_t *needed)
 {
 	struct trace_output_hdr *output_hdr = NULL;
 	void *end, *ptr = buff;
-	int rec, upto;
-	int count;
+	size_t rec, upto;
+	size_t count;
 
 	end = buff ? buff + buff_size : NULL;
 
@@ -284,6 +331,8 @@ int __attribute__((no_instrument_function)) trace_init(void *buff,
 	size_t needed;
 	int was_disabled = !trace_enabled;
 
+	trace_save_gd();
+
 	if (!was_disabled) {
 #ifdef CONFIG_TRACE_EARLY
 		char *end;
@@ -327,7 +376,7 @@ int __attribute__((no_instrument_function)) trace_init(void *buff,
 	add_textbase();
 
 	puts("trace: enabled\n");
-	hdr->depth_limit = 15;
+	hdr->depth_limit = CONFIG_TRACE_CALL_DEPTH_LIMIT;
 	trace_enabled = 1;
 	trace_inited = 1;
 
@@ -361,7 +410,7 @@ int __attribute__((no_instrument_function)) trace_early_init(void)
 	hdr->ftrace = (struct trace_call *)((char *)hdr + needed);
 	hdr->ftrace_size = (buff_size - needed) / sizeof(*hdr->ftrace);
 	add_textbase();
-	hdr->depth_limit = 200;
+	hdr->depth_limit = CONFIG_TRACE_EARLY_CALL_DEPTH_LIMIT;
 	printf("trace: early enable at %08x\n", CONFIG_TRACE_EARLY_ADDR);
 
 	trace_enabled = 1;

test/dm/pci.c

@@ -245,3 +245,52 @@ static int dm_test_pci_cap(struct unit_test_state *uts)
 	return 0;
 }
 DM_TEST(dm_test_pci_cap, DM_TESTF_SCAN_PDATA | DM_TESTF_SCAN_FDT);
+
+/* Test looking up BARs in EA capability structure */
+static int dm_test_pci_ea(struct unit_test_state *uts)
+{
+	struct udevice *bus, *swap;
+	void *bar;
+	int cap;
+
+	/*
+	 * use emulated device mapping function, we're not using real physical
+	 * addresses in this test
+	 */
+	sandbox_set_enable_pci_map(true);
+
+	ut_assertok(uclass_get_device_by_seq(UCLASS_PCI, 0, &bus));
+	ut_assertok(dm_pci_bus_find_bdf(PCI_BDF(0, 0x01, 0), &swap));
+
+	/* look up PCI_CAP_ID_EA */
+	cap = dm_pci_find_capability(swap, PCI_CAP_ID_EA);
+	ut_asserteq(PCI_CAP_ID_EA_OFFSET, cap);
+
+	/* test swap case in BAR 1 */
+	bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_0, 0);
+	ut_assertnonnull(bar);
+	*(int *)bar = 2; /* swap upper/lower */
+
+	bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_1, 0);
+	ut_assertnonnull(bar);
+	strcpy(bar, "ea TEST");
+	unmap_sysmem(bar);
+	bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_1, 0);
+	ut_assertnonnull(bar);
+	ut_asserteq_str("EA test", bar);
+
+	/* test magic values in BARs2, 4;  BAR 3 is n/a */
+	bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_2, 0);
+	ut_assertnonnull(bar);
+	ut_asserteq(PCI_EA_BAR2_MAGIC, *(u32 *)bar);
+
+	bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_3, 0);
+	ut_assertnull(bar);
+
+	bar = dm_pci_map_bar(swap, PCI_BASE_ADDRESS_4, 0);
+	ut_assertnonnull(bar);
+	ut_asserteq(PCI_EA_BAR4_MAGIC, *(u32 *)bar);
+
+	return 0;
+}
+DM_TEST(dm_test_pci_ea, DM_TESTF_SCAN_PDATA | DM_TESTF_SCAN_FDT);

test/py/tests/test_ofplatdata.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2016 Google, Inc
 
 import pytest
+import u_boot_utils as util
 
 OF_PLATDATA_OUTPUT = '''
 of-platdata probe:
@@ -31,6 +32,15 @@ intarray 0 0 0 0
 longbytearray 00 00 00 00 00 00 00 00 00
 string <NULL>
 stringarray "one" "" ""
+of-platdata probe:
+bool 0
+byte 00
+bytearray 00 00 00
+int 0
+intarray 0 0 0 0
+longbytearray 00 00 00 00 00 00 00 00 00
+string <NULL>
+stringarray "spl" "" ""
 '''
 
 @pytest.mark.buildconfigspec('spl_of_platdata')
@@ -40,3 +50,21 @@ def test_ofplatdata(u_boot_console):
     cons.restart_uboot_with_flags(['--show_of_platdata'])
     output = cons.get_spawn_output().replace('\r', '')
     assert OF_PLATDATA_OUTPUT in output
+
+@pytest.mark.buildconfigspec('spl_of_platdata')
+def test_spl_devicetree(u_boot_console):
+    """Test content of spl device-tree"""
+    cons = u_boot_console
+    dtb = cons.config.build_dir + '/spl/u-boot-spl.dtb'
+    fdtgrep = cons.config.build_dir + '/tools/fdtgrep'
+    output = util.run_and_log(cons, [fdtgrep, '-l', dtb])
+
+    assert "u-boot,dm-pre-reloc" not in output
+    assert "u-boot,dm-pre-proper" not in output
+    assert "u-boot,dm-spl" not in output
+    assert "u-boot,dm-tpl" not in output
+
+    assert "spl-test4" in output
+    assert "spl-test5" not in output
+    assert "spl-test6" not in output
+    assert "spl-test7" in output

tools/binman/README

@@ -702,6 +702,20 @@ To enable Python test coverage on Debian-type distributions (e.g. Ubuntu):
    $ sudo apt-get install python-coverage python-pytest
 
 
+Concurrent tests
+----------------
+
+Binman tries to run tests concurrently. This means that the tests make use of
+all available CPUs to run.
+
+ To enable this:
+
+   $ sudo apt-get install python-subunit python3-subunit
+
+Use '-P 1' to disable this. It is automatically disabled when code coverage is
+being used (-T) since they are incompatible.
+
+
 Advanced Features / Technical docs
 ----------------------------------
 

tools/binman/README.entries

@@ -224,6 +224,20 @@ See README.x86 for information about x86 binary blobs.
 
 
 
+Entry: intel-refcode: Entry containing an Intel Reference Code file
+-------------------------------------------------------------------
+
+Properties / Entry arguments:
+    - filename: Filename of file to read into entry
+
+This file contains code for setting up the platform on some Intel systems.
+This is executed by U-Boot when needed early during startup. A typical
+filename is 'refcode.bin'.
+
+See README.x86 for information about x86 binary blobs.
+
+
+
 Entry: intel-vbt: Entry containing an Intel Video BIOS Table (VBT) file
 -----------------------------------------------------------------------
 
@@ -627,6 +641,7 @@ Entry: vblock: An entry which contains a Chromium OS verified boot block
 ------------------------------------------------------------------------
 
 Properties / Entry arguments:
+    - content: List of phandles to entries to sign
     - keydir: Directory containing the public keys to use
     - keyblock: Name of the key file to use (inside keydir)
     - signprivate: Name of provide key file to use (inside keydir)

tools/binman/binman.py

@@ -9,6 +9,8 @@
 
 """See README for more information"""
 
+from __future__ import print_function
+
 import glob
 import multiprocessing
 import os
@@ -85,13 +87,25 @@ def RunTests(debug, processes, args):
     else:
         suite.run(result)
 
-    print result
+    # Remove errors which just indicate a missing test. Since Python v3.5 If an
+    # ImportError or AttributeError occurs while traversing name then a
+    # synthetic test that raises that error when run will be returned. These
+    # errors are included in the errors accumulated by result.errors.
+    if test_name:
+        errors = []
+        for test, err in result.errors:
+            if ("has no attribute '%s'" % test_name) not in err:
+                errors.append((test, err))
+            result.testsRun -= 1
+        result.errors = errors
+
+    print(result)
     for test, err in result.errors:
-        print test.id(), err
+        print(test.id(), err)
     for test, err in result.failures:
-        print err, result.failures
+        print(err, result.failures)
     if result.errors or result.failures:
-      print 'binman tests FAILED'
+      print('binman tests FAILED')
       return 1
     return 0
 
@@ -143,9 +157,9 @@ def RunBinman(options, args):
         try:
             ret_code = control.Binman(options, args)
         except Exception as e:
-            print 'binman: %s' % e
+            print('binman: %s' % e)
             if options.debug:
-                print
+                print()
                 traceback.print_exc()
             ret_code = 1
     return ret_code

tools/binman/bsection.py

@@ -8,7 +8,6 @@
 from __future__ import print_function
 
 from collections import OrderedDict
-from sets import Set
 import sys
 
 import fdt_util
@@ -109,7 +108,7 @@ class Section(object):
 
     def GetFdtSet(self):
         """Get the set of device tree files used by this image"""
-        fdt_set = Set()
+        fdt_set = set()
         for entry in self._entries.values():
             fdt_set.update(entry.GetFdtSet())
         return fdt_set
@@ -254,7 +253,7 @@ class Section(object):
         """
         for entry in self._entries.values():
             offset_dict = entry.GetOffsets()
-            for name, info in offset_dict.iteritems():
+            for name, info in offset_dict.items():
                 self._SetEntryOffsetSize(name, *info)
 
     def PackEntries(self):
@@ -333,7 +332,7 @@ class Section(object):
 
     def GetData(self):
         """Get the contents of the section"""
-        section_data = chr(self._pad_byte) * self._size
+        section_data = tools.GetBytes(self._pad_byte, self._size)
 
         for entry in self._entries.values():
             data = entry.GetData()

tools/binman/control.py

@@ -5,6 +5,8 @@
 # Creates binary images from input files controlled by a description
 #
 
+from __future__ import print_function
+
 from collections import OrderedDict
 import os
 import sys
@@ -129,12 +131,15 @@ def Binman(options, args):
 
             if options.image:
                 skip = []
-                for name, image in images.iteritems():
-                    if name not in options.image:
-                        del images[name]
+                new_images = OrderedDict()
+                for name, image in images.items():
+                    if name in options.image:
+                        new_images[name] = image
+                    else:
                         skip.append(name)
+                images = new_images
                 if skip and options.verbosity >= 2:
-                    print 'Skipping images: %s' % ', '.join(skip)
+                    print('Skipping images: %s' % ', '.join(skip))
 
             state.Prepare(images, dtb)
 
@@ -170,7 +175,7 @@ def Binman(options, args):
                 except Exception as e:
                     if options.map:
                         fname = image.WriteMap()
-                        print "Wrote map file '%s' to show errors"  % fname
+                        print("Wrote map file '%s' to show errors"  % fname)
                     raise
                 image.SetImagePos()
                 if options.update_fdt:

tools/binman/elf.py

@@ -59,7 +59,7 @@ def GetSymbols(fname, patterns):
                                 flags[1] == 'w')
 
     # Sort dict by address
-    return OrderedDict(sorted(syms.iteritems(), key=lambda x: x[1].address))
+    return OrderedDict(sorted(syms.items(), key=lambda x: x[1].address))
 
 def GetSymbolAddress(fname, sym_name):
     """Get a value of a symbol from an ELF file
@@ -98,7 +98,7 @@ def LookupAndWriteSymbols(elf_fname, entry, section):
     base = syms.get('__image_copy_start')
     if not base:
         return
-    for name, sym in syms.iteritems():
+    for name, sym in syms.items():
         if name.startswith('_binman'):
             msg = ("Section '%s': Symbol '%s'\n   in entry '%s'" %
                    (section.GetPath(), name, entry.GetPath()))

tools/binman/elf_test.py

@@ -22,7 +22,7 @@ class FakeEntry:
     """
     def __init__(self, contents_size):
         self.contents_size = contents_size
-        self.data = 'a' * contents_size
+        self.data = tools.GetBytes(ord('a'), contents_size)
 
     def GetPath(self):
         return 'entry_path'
@@ -122,7 +122,8 @@ class TestElf(unittest.TestCase):
         section = FakeSection(sym_value=None)
         elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms')
         syms = elf.LookupAndWriteSymbols(elf_fname, entry, section)
-        self.assertEqual(chr(255) * 16 + 'a' * 4, entry.data)
+        self.assertEqual(tools.GetBytes(255, 16) + tools.GetBytes(ord('a'), 4),
+                                                                  entry.data)
 
     def testDebug(self):
         """Check that enabling debug in the elf module produced debug output"""

tools/binman/entry.py

@@ -18,7 +18,6 @@ except:
     have_importlib = False
 
 import os
-from sets import Set
 import sys
 
 import fdt_util
@@ -178,8 +177,8 @@ class Entry(object):
         # It would be better to use isinstance(self, Entry_blob_dtb) here but
         # we cannot access Entry_blob_dtb
         if fname and fname.endswith('.dtb'):
-            return Set([fname])
-        return Set()
+            return set([fname])
+        return set()
 
     def ExpandEntries(self):
         pass

tools/binman/entry_test.py

@@ -41,6 +41,10 @@ class TestEntry(unittest.TestCase):
         del sys.modules['importlib']
         global entry
         if entry:
+            if sys.version_info[0] >= 3:
+                import importlib
+                importlib.reload(entry)
+            else:
                 reload(entry)
         else:
             import entry

tools/binman/etype/_testing.py

@@ -75,7 +75,7 @@ class Entry__testing(Entry):
     def ObtainContents(self):
         if self.return_unknown_contents or not self.return_contents:
             return False
-        self.data = 'a'
+        self.data = b'a'
         self.contents_size = len(self.data)
         if self.return_contents_once:
             self.return_contents = False

tools/binman/etype/blob.py

@@ -60,7 +60,7 @@ class Entry_blob(Entry):
             except AttributeError:
                 data = lz4.compress(data)
             '''
-            data = tools.Run('lz4', '-c', self._pathname)
+            data = tools.Run('lz4', '-c', self._pathname, binary=True)
         self.SetContents(data)
         return True
 

tools/binman/etype/fill.py

@@ -5,7 +5,7 @@
 
 from entry import Entry
 import fdt_util
-
+import tools
 
 class Entry_fill(Entry):
     """An entry which is filled to a particular byte value
@@ -28,5 +28,5 @@ class Entry_fill(Entry):
         self.fill_value = fdt_util.GetByte(self._node, 'fill-byte', 0)
 
     def ObtainContents(self):
-        self.SetContents(chr(self.fill_value) * self.size)
+        self.SetContents(tools.GetBytes(self.fill_value, self.size))
         return True

tools/binman/etype/fmap.py

@@ -7,6 +7,7 @@
 
 from entry import Entry
 import fmap_util
+import tools
 
 
 class Entry_fmap(Entry):
@@ -46,7 +47,7 @@ class Entry_fmap(Entry):
                 if pos is not None:
                     pos -= entry.section.GetRootSkipAtStart()
                 areas.append(fmap_util.FmapArea(pos or 0, entry.size or 0,
-                                                entry.name, 0))
+                                            tools.FromUnicode(entry.name), 0))
 
         entries = self.section._image.GetEntries()
         areas = []

tools/binman/etype/gbb.py

@@ -64,7 +64,7 @@ class Entry_gbb(Entry):
         self.gbb_flags = 0
         flags_node = node.FindNode('flags')
         if flags_node:
-            for flag, value in gbb_flag_properties.iteritems():
+            for flag, value in gbb_flag_properties.items():
                 if fdt_util.GetBool(flags_node, flag):
                     self.gbb_flags |= value
 

tools/binman/etype/text.py

@@ -7,6 +7,7 @@ from collections import OrderedDict
 
 from entry import Entry, EntryArg
 import fdt_util
+import tools
 
 
 class Entry_text(Entry):
@@ -48,9 +49,11 @@ class Entry_text(Entry):
     """
     def __init__(self, section, etype, node):
         Entry.__init__(self, section, etype, node)
-        self.text_label, = self.GetEntryArgsOrProps(
-            [EntryArg('text-label', str)])
-        self.value, = self.GetEntryArgsOrProps([EntryArg(self.text_label, str)])
+        label, = self.GetEntryArgsOrProps([EntryArg('text-label', str)])
+        self.text_label = tools.ToStr(label) if type(label) != str else label
+        value, = self.GetEntryArgsOrProps([EntryArg(self.text_label, str)])
+        value = tools.ToBytes(value) if value is not None else value
+        self.value = value
 
     def ObtainContents(self):
         if not self.value:

tools/binman/etype/u_boot_dtb_with_ucode.py

@@ -26,7 +26,7 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb):
     """
     def __init__(self, section, etype, node):
         Entry_blob_dtb.__init__(self, section, etype, node)
-        self.ucode_data = ''
+        self.ucode_data = b''
         self.collate = False
         self.ucode_offset = None
         self.ucode_size = None
@@ -65,7 +65,7 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb):
         for node in self.ucode.subnodes:
             data_prop = node.props.get('data')
             if data_prop:
-                self.ucode_data += ''.join(data_prop.bytes)
+                self.ucode_data += data_prop.bytes
                 if self.collate:
                     node.DeleteProp('data')
         return True

tools/binman/etype/u_boot_spl_bss_pad.py

@@ -38,5 +38,5 @@ class Entry_u_boot_spl_bss_pad(Entry_blob):
         bss_size = elf.GetSymbolAddress(fname, '__bss_size')
         if not bss_size:
             self.Raise('Expected __bss_size symbol in spl/u-boot-spl')
-        self.SetContents(chr(0) * bss_size)
+        self.SetContents(tools.GetBytes(0, bss_size))
         return True

tools/binman/etype/u_boot_ucode.py

@@ -69,7 +69,7 @@ class Entry_u_boot_ucode(Entry_blob):
             if entry and entry.target_offset:
                 found = True
         if not found:
-            self.data = ''
+            self.data = b''
             return True
         # Get the microcode from the device tree entry. If it is not available
         # yet, return False so we will be called later. If the section simply
@@ -87,7 +87,7 @@ class Entry_u_boot_ucode(Entry_blob):
 
         if not fdt_entry.collate:
             # This binary can be empty
-            self.data = ''
+            self.data = b''
             return True
 
         # Write it out to a file

tools/binman/etype/vblock.py

@@ -51,7 +51,7 @@ class Entry_vblock(Entry):
 
     def ObtainContents(self):
         # Join up the data files to be signed
-        input_data = ''
+        input_data = b''
         for entry_phandle in self.content:
             data = self.section.GetContentsByPhandle(entry_phandle, self)
             if data is None:

tools/binman/fmap_util.py

@@ -8,9 +8,12 @@
 
 import collections
 import struct
+import sys
+
+import tools
 
 # constants imported from lib/fmap.h
-FMAP_SIGNATURE = '__FMAP__'
+FMAP_SIGNATURE = b'__FMAP__'
 FMAP_VER_MAJOR = 1
 FMAP_VER_MINOR = 0
 FMAP_STRLEN = 32
@@ -50,6 +53,8 @@ FmapArea = collections.namedtuple('FmapArea', FMAP_AREA_NAMES)
 
 
 def NameToFmap(name):
+    if type(name) == bytes and sys.version_info[0] >= 3:
+        name = name.decode('utf-8')  # pragma: no cover (for Python 2)
     return name.replace('\0', '').replace('-', '_').upper()
 
 def ConvertName(field_names, fields):
@@ -65,7 +70,7 @@ def ConvertName(field_names, fields):
             value: value of that field (string for the ones we support)
     """
     name_index = field_names.index('name')
-    fields[name_index] = NameToFmap(fields[name_index])
+    fields[name_index] = tools.ToBytes(NameToFmap(fields[name_index]))
 
 def DecodeFmap(data):
     """Decode a flashmap into a header and list of areas
@@ -106,7 +111,8 @@ def EncodeFmap(image_size, name, areas):
         ConvertName(names, params)
         return struct.pack(fmt, *params)
 
-    values = FmapHeader(FMAP_SIGNATURE, 1, 0, 0, image_size, name, len(areas))
+    values = FmapHeader(FMAP_SIGNATURE, 1, 0, 0, image_size,
+                        tools.FromUnicode(name), len(areas))
     blob = _FormatBlob(FMAP_HEADER_FORMAT, FMAP_HEADER_NAMES, values)
     for area in areas:
         blob += _FormatBlob(FMAP_AREA_FORMAT, FMAP_AREA_NAMES, area)

tools/binman/ftest.py

@@ -29,38 +29,38 @@ import tools
 import tout
 
 # Contents of test files, corresponding to different entry types
-U_BOOT_DATA           = '1234'
-U_BOOT_IMG_DATA       = 'img'
-U_BOOT_SPL_DATA       = '56780123456789abcde'
-U_BOOT_TPL_DATA       = 'tpl'
-BLOB_DATA             = '89'
-ME_DATA               = '0abcd'
-VGA_DATA              = 'vga'
-U_BOOT_DTB_DATA       = 'udtb'
-U_BOOT_SPL_DTB_DATA   = 'spldtb'
-U_BOOT_TPL_DTB_DATA   = 'tpldtb'
-X86_START16_DATA      = 'start16'
-X86_START16_SPL_DATA  = 'start16spl'
-X86_START16_TPL_DATA  = 'start16tpl'
-PPC_MPC85XX_BR_DATA   = 'ppcmpc85xxbr'
-U_BOOT_NODTB_DATA     = 'nodtb with microcode pointer somewhere in here'
-U_BOOT_SPL_NODTB_DATA = 'splnodtb with microcode pointer somewhere in here'
-U_BOOT_TPL_NODTB_DATA = 'tplnodtb with microcode pointer somewhere in here'
-FSP_DATA              = 'fsp'
-CMC_DATA              = 'cmc'
-VBT_DATA              = 'vbt'
-MRC_DATA              = 'mrc'
+U_BOOT_DATA           = b'1234'
+U_BOOT_IMG_DATA       = b'img'
+U_BOOT_SPL_DATA       = b'56780123456789abcde'
+U_BOOT_TPL_DATA       = b'tpl'
+BLOB_DATA             = b'89'
+ME_DATA               = b'0abcd'
+VGA_DATA              = b'vga'
+U_BOOT_DTB_DATA       = b'udtb'
+U_BOOT_SPL_DTB_DATA   = b'spldtb'
+U_BOOT_TPL_DTB_DATA   = b'tpldtb'
+X86_START16_DATA      = b'start16'
+X86_START16_SPL_DATA  = b'start16spl'
+X86_START16_TPL_DATA  = b'start16tpl'
+PPC_MPC85XX_BR_DATA   = b'ppcmpc85xxbr'
+U_BOOT_NODTB_DATA     = b'nodtb with microcode pointer somewhere in here'
+U_BOOT_SPL_NODTB_DATA = b'splnodtb with microcode pointer somewhere in here'
+U_BOOT_TPL_NODTB_DATA = b'tplnodtb with microcode pointer somewhere in here'
+FSP_DATA              = b'fsp'
+CMC_DATA              = b'cmc'
+VBT_DATA              = b'vbt'
+MRC_DATA              = b'mrc'
 TEXT_DATA             = 'text'
 TEXT_DATA2            = 'text2'
 TEXT_DATA3            = 'text3'
-CROS_EC_RW_DATA       = 'ecrw'
-GBB_DATA              = 'gbbd'
-BMPBLK_DATA           = 'bmp'
-VBLOCK_DATA           = 'vblk'
-FILES_DATA            = ("sorry I'm late\nOh, don't bother apologising, I'm " +
-                         "sorry you're alive\n")
-COMPRESS_DATA         = 'data to compress'
-REFCODE_DATA          = 'refcode'
+CROS_EC_RW_DATA       = b'ecrw'
+GBB_DATA              = b'gbbd'
+BMPBLK_DATA           = b'bmp'
+VBLOCK_DATA           = b'vblk'
+FILES_DATA            = (b"sorry I'm late\nOh, don't bother apologising, I'm " +
+                         b"sorry you're alive\n")
+COMPRESS_DATA         = b'data to compress'
+REFCODE_DATA          = b'refcode'
 
 
 class TestFunctional(unittest.TestCase):
@@ -119,11 +119,11 @@ class TestFunctional(unittest.TestCase):
         TestFunctional._MakeInputFile('refcode.bin', REFCODE_DATA)
 
         # ELF file with a '_dt_ucode_base_size' symbol
-        with open(self.TestFile('u_boot_ucode_ptr')) as fd:
+        with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd:
             TestFunctional._MakeInputFile('u-boot', fd.read())
 
         # Intel flash descriptor file
-        with open(self.TestFile('descriptor.bin')) as fd:
+        with open(self.TestFile('descriptor.bin'), 'rb') as fd:
             TestFunctional._MakeInputFile('descriptor.bin', fd.read())
 
         shutil.copytree(self.TestFile('files'),
@@ -214,7 +214,7 @@ class TestFunctional(unittest.TestCase):
         if verbosity is not None:
             args.append('-v%d' % verbosity)
         if entry_args:
-            for arg, value in entry_args.iteritems():
+            for arg, value in entry_args.items():
                 args.append('-a%s=%s' % (arg, value))
         if images:
             for image in images:
@@ -236,7 +236,7 @@ class TestFunctional(unittest.TestCase):
         """
         tools.PrepareOutputDir(None)
         dtb = fdt_util.EnsureCompiled(self.TestFile(fname))
-        with open(dtb) as fd:
+        with open(dtb, 'rb') as fd:
             data = fd.read()
             TestFunctional._MakeInputFile(outfile, data)
         tools.FinaliseOutputDir()
@@ -291,7 +291,6 @@ class TestFunctional(unittest.TestCase):
         # Use the compiled test file as the u-boot-dtb input
         if use_real_dtb:
             dtb_data = self._SetupDtb(fname)
-            infile = os.path.join(self._indir, 'u-boot.dtb')
 
             # For testing purposes, make a copy of the DT for SPL and TPL. Add
             # a node indicating which it is, so aid verification.
@@ -317,7 +316,7 @@ class TestFunctional(unittest.TestCase):
                     map_data = fd.read()
             else:
                 map_data = None
-            with open(image_fname) as fd:
+            with open(image_fname, 'rb') as fd:
                 return fd.read(), dtb_data, map_data, out_dtb_fname
         finally:
             # Put the test file back
@@ -379,7 +378,7 @@ class TestFunctional(unittest.TestCase):
         Args:
             Filename of ELF file to use as SPL
         """
-        with open(self.TestFile(src_fname)) as fd:
+        with open(self.TestFile(src_fname), 'rb') as fd:
             TestFunctional._MakeInputFile('spl/u-boot-spl', fd.read())
 
     @classmethod
@@ -396,7 +395,7 @@ class TestFunctional(unittest.TestCase):
         for grep in grep_list:
             if grep in target:
                 return
-        self.fail("Error: '%' not found in '%s'" % (grep_list, target))
+        self.fail("Error: '%s' not found in '%s'" % (grep_list, target))
 
     def CheckNoGaps(self, entries):
         """Check that all entries fit together without gaps
@@ -541,7 +540,7 @@ class TestFunctional(unittest.TestCase):
         self.assertEqual(len(U_BOOT_DATA), image._size)
         fname = tools.GetOutputFilename('image1.bin')
         self.assertTrue(os.path.exists(fname))
-        with open(fname) as fd:
+        with open(fname, 'rb') as fd:
             data = fd.read()
             self.assertEqual(U_BOOT_DATA, data)
 
@@ -549,11 +548,11 @@ class TestFunctional(unittest.TestCase):
         self.assertEqual(3 + len(U_BOOT_DATA) + 5, image._size)
         fname = tools.GetOutputFilename('image2.bin')
         self.assertTrue(os.path.exists(fname))
-        with open(fname) as fd:
+        with open(fname, 'rb') as fd:
             data = fd.read()
             self.assertEqual(U_BOOT_DATA, data[3:7])
-            self.assertEqual(chr(0) * 3, data[:3])
-            self.assertEqual(chr(0) * 5, data[7:])
+            self.assertEqual(tools.GetBytes(0, 3), data[:3])
+            self.assertEqual(tools.GetBytes(0, 5), data[7:])
 
     def testBadAlign(self):
         """Test that an invalid alignment value is detected"""
@@ -732,7 +731,8 @@ class TestFunctional(unittest.TestCase):
         """Test that the image pad byte can be specified"""
         self._SetupSplElf()
         data = self._DoReadFile('021_image_pad.dts')
-        self.assertEqual(U_BOOT_SPL_DATA + (chr(0xff) * 1) + U_BOOT_DATA, data)
+        self.assertEqual(U_BOOT_SPL_DATA + tools.GetBytes(0xff, 1) +
+                         U_BOOT_DATA, data)
 
     def testImageName(self):
         """Test that image files can be named"""
@@ -755,8 +755,8 @@ class TestFunctional(unittest.TestCase):
         """Test that entries can be sorted"""
         self._SetupSplElf()
         data = self._DoReadFile('024_sorted.dts')
-        self.assertEqual(chr(0) * 1 + U_BOOT_SPL_DATA + chr(0) * 2 +
-                         U_BOOT_DATA, data)
+        self.assertEqual(tools.GetBytes(0, 1) + U_BOOT_SPL_DATA +
+                         tools.GetBytes(0, 2) + U_BOOT_DATA, data)
 
     def testPackZeroOffset(self):
         """Test that an entry at offset 0 is not given a new offset"""
@@ -798,12 +798,12 @@ class TestFunctional(unittest.TestCase):
         """Test that a basic x86 ROM can be created"""
         self._SetupSplElf()
         data = self._DoReadFile('029_x86-rom.dts')
-        self.assertEqual(U_BOOT_DATA + chr(0) * 7 + U_BOOT_SPL_DATA +
-                         chr(0) * 2, data)
+        self.assertEqual(U_BOOT_DATA + tools.GetBytes(0, 7) + U_BOOT_SPL_DATA +
+                         tools.GetBytes(0, 2), data)
 
     def testPackX86RomMeNoDesc(self):
         """Test that an invalid Intel descriptor entry is detected"""
-        TestFunctional._MakeInputFile('descriptor.bin', '')
+        TestFunctional._MakeInputFile('descriptor.bin', b'')
         with self.assertRaises(ValueError) as e:
             self._DoTestFile('031_x86-rom-me.dts')
         self.assertIn("Node '/binman/intel-descriptor': Cannot find FD "
@@ -900,8 +900,8 @@ class TestFunctional(unittest.TestCase):
         """
         first, pos_and_size = self._RunMicrocodeTest('034_x86_ucode.dts',
                                                      U_BOOT_NODTB_DATA)
-        self.assertEqual('nodtb with microcode' + pos_and_size +
-                         ' somewhere in here', first)
+        self.assertEqual(b'nodtb with microcode' + pos_and_size +
+                         b' somewhere in here', first)
 
     def _RunPackUbootSingleMicrocode(self):
         """Test that x86 microcode can be handled correctly
@@ -932,8 +932,8 @@ class TestFunctional(unittest.TestCase):
         pos_and_size = struct.pack('<2L', 0xfffffe00 + ucode_pos,
                                    len(ucode_data))
         first = data[:len(U_BOOT_NODTB_DATA)]
-        self.assertEqual('nodtb with microcode' + pos_and_size +
-                         ' somewhere in here', first)
+        self.assertEqual(b'nodtb with microcode' + pos_and_size +
+                         b' somewhere in here', first)
 
     def testPackUbootSingleMicrocode(self):
         """Test that x86 microcode can be handled correctly with fdt_normal.
@@ -970,7 +970,7 @@ class TestFunctional(unittest.TestCase):
         """Test that a U-Boot binary without the microcode symbol is detected"""
         # ELF file without a '_dt_ucode_base_size' symbol
         try:
-            with open(self.TestFile('u_boot_no_ucode_ptr')) as fd:
+            with open(self.TestFile('u_boot_no_ucode_ptr'), 'rb') as fd:
                 TestFunctional._MakeInputFile('u-boot', fd.read())
 
             with self.assertRaises(ValueError) as e:
@@ -980,7 +980,7 @@ class TestFunctional(unittest.TestCase):
 
         finally:
             # Put the original file back
-            with open(self.TestFile('u_boot_ucode_ptr')) as fd:
+            with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd:
                 TestFunctional._MakeInputFile('u-boot', fd.read())
 
     def testMicrocodeNotInImage(self):
@@ -993,7 +993,7 @@ class TestFunctional(unittest.TestCase):
 
     def testWithoutMicrocode(self):
         """Test that we can cope with an image without microcode (e.g. qemu)"""
-        with open(self.TestFile('u_boot_no_ucode_ptr')) as fd:
+        with open(self.TestFile('u_boot_no_ucode_ptr'), 'rb') as fd:
             TestFunctional._MakeInputFile('u-boot', fd.read())
         data, dtb, _, _ = self._DoReadFileDtb('044_x86_optional_ucode.dts', True)
 
@@ -1006,7 +1006,7 @@ class TestFunctional(unittest.TestCase):
 
         used_len = len(U_BOOT_NODTB_DATA) + fdt_len
         third = data[used_len:]
-        self.assertEqual(chr(0) * (0x200 - used_len), third)
+        self.assertEqual(tools.GetBytes(0, 0x200 - used_len), third)
 
     def testUnknownPosSize(self):
         """Test that microcode must be placed within the image"""
@@ -1035,7 +1035,8 @@ class TestFunctional(unittest.TestCase):
         # ELF file with a '__bss_size' symbol
         self._SetupSplElf()
         data = self._DoReadFile('047_spl_bss_pad.dts')
-        self.assertEqual(U_BOOT_SPL_DATA + (chr(0) * 10) + U_BOOT_DATA, data)
+        self.assertEqual(U_BOOT_SPL_DATA + tools.GetBytes(0, 10) + U_BOOT_DATA,
+                         data)
 
     def testSplBssPadMissing(self):
         """Test that a missing symbol is detected"""
@@ -1067,8 +1068,8 @@ class TestFunctional(unittest.TestCase):
         self._SetupSplElf('u_boot_ucode_ptr')
         first, pos_and_size = self._RunMicrocodeTest(dts, U_BOOT_SPL_NODTB_DATA,
                                                      ucode_second=ucode_second)
-        self.assertEqual('splnodtb with microc' + pos_and_size +
-                         'ter somewhere in here', first)
+        self.assertEqual(b'splnodtb with microc' + pos_and_size +
+                         b'ter somewhere in here', first)
 
     def testPackUbootSplMicrocode(self):
         """Test that x86 microcode can be handled correctly in SPL"""
@@ -1109,9 +1110,9 @@ class TestFunctional(unittest.TestCase):
         self._SetupSplElf('u_boot_binman_syms')
         data = self._DoReadFile('053_symbols.dts')
         sym_values = struct.pack('<LQL', 0x24 + 0, 0x24 + 24, 0x24 + 20)
-        expected = (sym_values + U_BOOT_SPL_DATA[16:] + chr(0xff) +
-                    U_BOOT_DATA +
-                    sym_values + U_BOOT_SPL_DATA[16:])
+        expected = (sym_values + U_BOOT_SPL_DATA[16:] +
+                    tools.GetBytes(0xff, 1) + U_BOOT_DATA + sym_values +
+                    U_BOOT_SPL_DATA[16:])
         self.assertEqual(expected, data)
 
     def testPackUnitAddress(self):
@@ -1122,8 +1123,9 @@ class TestFunctional(unittest.TestCase):
     def testSections(self):
         """Basic test of sections"""
         data = self._DoReadFile('055_sections.dts')
-        expected = (U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12 +
-                    U_BOOT_DATA + '&' * 4)
+        expected = (U_BOOT_DATA + tools.GetBytes(ord('!'), 12) +
+                    U_BOOT_DATA + tools.GetBytes(ord('a'), 12) +
+                    U_BOOT_DATA + tools.GetBytes(ord('&'), 4))
         self.assertEqual(expected, data)
 
     def testMap(self):
@@ -1281,8 +1283,10 @@ class TestFunctional(unittest.TestCase):
         }
         data, _, _, _ = self._DoReadFileDtb('066_text.dts',
                                             entry_args=entry_args)
-        expected = (TEXT_DATA + chr(0) * (8 - len(TEXT_DATA)) + TEXT_DATA2 +
-                    TEXT_DATA3 + 'some text')
+        expected = (tools.ToBytes(TEXT_DATA) +
+                    tools.GetBytes(0, 8 - len(TEXT_DATA)) +
+                    tools.ToBytes(TEXT_DATA2) + tools.ToBytes(TEXT_DATA3) +
+                    b'some text')
         self.assertEqual(expected, data)
 
     def testEntryDocs(self):
@@ -1303,32 +1307,33 @@ class TestFunctional(unittest.TestCase):
         """Basic test of generation of a flashrom fmap"""
         data = self._DoReadFile('067_fmap.dts')
         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
-        expected = U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12
+        expected = (U_BOOT_DATA + tools.GetBytes(ord('!'), 12) +
+                    U_BOOT_DATA + tools.GetBytes(ord('a'), 12))
         self.assertEqual(expected, data[:32])
-        self.assertEqual('__FMAP__', fhdr.signature)
+        self.assertEqual(b'__FMAP__', fhdr.signature)
         self.assertEqual(1, fhdr.ver_major)
         self.assertEqual(0, fhdr.ver_minor)
         self.assertEqual(0, fhdr.base)
         self.assertEqual(16 + 16 +
                          fmap_util.FMAP_HEADER_LEN +
                          fmap_util.FMAP_AREA_LEN * 3, fhdr.image_size)
-        self.assertEqual('FMAP', fhdr.name)
+        self.assertEqual(b'FMAP', fhdr.name)
         self.assertEqual(3, fhdr.nareas)
         for fentry in fentries:
             self.assertEqual(0, fentry.flags)
 
         self.assertEqual(0, fentries[0].offset)
         self.assertEqual(4, fentries[0].size)
-        self.assertEqual('RO_U_BOOT', fentries[0].name)
+        self.assertEqual(b'RO_U_BOOT', fentries[0].name)
 
         self.assertEqual(16, fentries[1].offset)
         self.assertEqual(4, fentries[1].size)
-        self.assertEqual('RW_U_BOOT', fentries[1].name)
+        self.assertEqual(b'RW_U_BOOT', fentries[1].name)
 
         self.assertEqual(32, fentries[2].offset)
         self.assertEqual(fmap_util.FMAP_HEADER_LEN +
                          fmap_util.FMAP_AREA_LEN * 3, fentries[2].size)
-        self.assertEqual('FMAP', fentries[2].name)
+        self.assertEqual(b'FMAP', fentries[2].name)
 
     def testBlobNamedByArg(self):
         """Test we can add a blob with the filename coming from an entry arg"""
@@ -1341,7 +1346,7 @@ class TestFunctional(unittest.TestCase):
     def testFill(self):
         """Test for an fill entry type"""
         data = self._DoReadFile('069_fill.dts')
-        expected = 8 * chr(0xff) + 8 * chr(0)
+        expected = tools.GetBytes(0xff, 8) + tools.GetBytes(0, 8)
         self.assertEqual(expected, data)
 
     def testFillNoSize(self):
@@ -1357,7 +1362,7 @@ class TestFunctional(unittest.TestCase):
             fname = pipe_list[0][-1]
             # Append our GBB data to the file, which will happen every time the
             # futility command is called.
-            with open(fname, 'a') as fd:
+            with open(fname, 'ab') as fd:
                 fd.write(GBB_DATA)
             return command.CommandResult()
 
@@ -1371,7 +1376,8 @@ class TestFunctional(unittest.TestCase):
         data, _, _, _ = self._DoReadFileDtb('071_gbb.dts', entry_args=entry_args)
 
         # Since futility
-        expected = GBB_DATA + GBB_DATA + 8 * chr(0) + (0x2180 - 16) * chr(0)
+        expected = (GBB_DATA + GBB_DATA + tools.GetBytes(0, 8) +
+                    tools.GetBytes(0, 0x2180 - 16))
         self.assertEqual(expected, data)
 
     def testGbbTooSmall(self):
@@ -1431,7 +1437,7 @@ class TestFunctional(unittest.TestCase):
     def testTpl(self):
         """Test that an image with TPL and ots device tree can be created"""
         # ELF file with a '__bss_size' symbol
-        with open(self.TestFile('bss_data')) as fd:
+        with open(self.TestFile('bss_data'), 'rb') as fd:
             TestFunctional._MakeInputFile('tpl/u-boot-tpl', fd.read())
         data = self._DoReadFile('078_u_boot_tpl.dts')
         self.assertEqual(U_BOOT_TPL_DATA + U_BOOT_TPL_DTB_DATA, data)
@@ -1446,7 +1452,7 @@ class TestFunctional(unittest.TestCase):
     def testFillZero(self):
         """Test for an fill entry type with a size of 0"""
         data = self._DoReadFile('080_fill_empty.dts')
-        self.assertEqual(chr(0) * 16, data)
+        self.assertEqual(tools.GetBytes(0, 16), data)
 
     def testTextMissing(self):
         """Test for a text entry type where there is no text"""
@@ -1557,7 +1563,7 @@ class TestFunctional(unittest.TestCase):
         out = os.path.join(self._indir, 'lz4.tmp')
         with open(out, 'wb') as fd:
             fd.write(data)
-        return tools.Run('lz4', '-dc', out)
+        return tools.Run('lz4', '-dc', out, binary=True)
         '''
         try:
             orig = lz4.frame.decompress(data)
@@ -1595,7 +1601,7 @@ class TestFunctional(unittest.TestCase):
         files = entries['files']
         entries = files._section._entries
 
-        orig = ''
+        orig = b''
         for i in range(1, 3):
             key = '%d.dat' % i
             start = entries[key].image_pos
@@ -1623,10 +1629,10 @@ class TestFunctional(unittest.TestCase):
         """Test an expanding entry"""
         data, _, map_data, _ = self._DoReadFileDtb('088_expand_size.dts',
                                                    map=True)
-        expect = ('a' * 8 + U_BOOT_DATA +
-                  MRC_DATA + 'b' * 1 + U_BOOT_DATA +
-                  'c' * 8 + U_BOOT_DATA +
-                  'd' * 8)
+        expect = (tools.GetBytes(ord('a'), 8) + U_BOOT_DATA +
+                  MRC_DATA + tools.GetBytes(ord('b'), 1) + U_BOOT_DATA +
+                  tools.GetBytes(ord('c'), 8) + U_BOOT_DATA +
+                  tools.GetBytes(ord('d'), 8))
         self.assertEqual(expect, data)
         self.assertEqual('''ImagePos    Offset      Size  Name
 00000000  00000000  00000028  main-section
@@ -1658,7 +1664,7 @@ class TestFunctional(unittest.TestCase):
         hash_node = dtb.GetNode('/binman/u-boot/hash').props['value']
         m = hashlib.sha256()
         m.update(U_BOOT_DATA)
-        self.assertEqual(m.digest(), ''.join(hash_node.value))
+        self.assertEqual(m.digest(), b''.join(hash_node.value))
 
     def testHashNoAlgo(self):
         with self.assertRaises(ValueError) as e:
@@ -1681,8 +1687,8 @@ class TestFunctional(unittest.TestCase):
         hash_node = dtb.GetNode('/binman/section/hash').props['value']
         m = hashlib.sha256()
         m.update(U_BOOT_DATA)
-        m.update(16 * 'a')
-        self.assertEqual(m.digest(), ''.join(hash_node.value))
+        m.update(tools.GetBytes(ord('a'), 16))
+        self.assertEqual(m.digest(), b''.join(hash_node.value))
 
     def testPackUBootTplMicrocode(self):
         """Test that x86 microcode can be handled correctly in TPL
@@ -1693,18 +1699,18 @@ class TestFunctional(unittest.TestCase):
             u-boot-tpl.dtb with the microcode removed
             the microcode
         """
-        with open(self.TestFile('u_boot_ucode_ptr')) as fd:
+        with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd:
             TestFunctional._MakeInputFile('tpl/u-boot-tpl', fd.read())
         first, pos_and_size = self._RunMicrocodeTest('093_x86_tpl_ucode.dts',
                                                      U_BOOT_TPL_NODTB_DATA)
-        self.assertEqual('tplnodtb with microc' + pos_and_size +
-                         'ter somewhere in here', first)
+        self.assertEqual(b'tplnodtb with microc' + pos_and_size +
+                         b'ter somewhere in here', first)
 
     def testFmapX86(self):
         """Basic test of generation of a flashrom fmap"""
         data = self._DoReadFile('094_fmap_x86.dts')
         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
-        expected = U_BOOT_DATA + MRC_DATA + 'a' * (32 - 7)
+        expected = U_BOOT_DATA + MRC_DATA + tools.GetBytes(ord('a'), 32 - 7)
         self.assertEqual(expected, data[:32])
         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
 
@@ -1712,21 +1718,21 @@ class TestFunctional(unittest.TestCase):
 
         self.assertEqual(0, fentries[0].offset)
         self.assertEqual(4, fentries[0].size)
-        self.assertEqual('U_BOOT', fentries[0].name)
+        self.assertEqual(b'U_BOOT', fentries[0].name)
 
         self.assertEqual(4, fentries[1].offset)
         self.assertEqual(3, fentries[1].size)
-        self.assertEqual('INTEL_MRC', fentries[1].name)
+        self.assertEqual(b'INTEL_MRC', fentries[1].name)
 
         self.assertEqual(32, fentries[2].offset)
         self.assertEqual(fmap_util.FMAP_HEADER_LEN +
                          fmap_util.FMAP_AREA_LEN * 3, fentries[2].size)
-        self.assertEqual('FMAP', fentries[2].name)
+        self.assertEqual(b'FMAP', fentries[2].name)
 
     def testFmapX86Section(self):
         """Basic test of generation of a flashrom fmap"""
         data = self._DoReadFile('095_fmap_x86_section.dts')
-        expected = U_BOOT_DATA + MRC_DATA + 'b' * (32 - 7)
+        expected = U_BOOT_DATA + MRC_DATA + tools.GetBytes(ord('b'), 32 - 7)
         self.assertEqual(expected, data[:32])
         fhdr, fentries = fmap_util.DecodeFmap(data[36:])
 
@@ -1734,28 +1740,28 @@ class TestFunctional(unittest.TestCase):
 
         self.assertEqual(0, fentries[0].offset)
         self.assertEqual(4, fentries[0].size)
-        self.assertEqual('U_BOOT', fentries[0].name)
+        self.assertEqual(b'U_BOOT', fentries[0].name)
 
         self.assertEqual(4, fentries[1].offset)
         self.assertEqual(3, fentries[1].size)
-        self.assertEqual('INTEL_MRC', fentries[1].name)
+        self.assertEqual(b'INTEL_MRC', fentries[1].name)
 
         self.assertEqual(36, fentries[2].offset)
         self.assertEqual(fmap_util.FMAP_HEADER_LEN +
                          fmap_util.FMAP_AREA_LEN * 3, fentries[2].size)
-        self.assertEqual('FMAP', fentries[2].name)
+        self.assertEqual(b'FMAP', fentries[2].name)
 
     def testElf(self):
         """Basic test of ELF entries"""
         self._SetupSplElf()
-        with open(self.TestFile('bss_data')) as fd:
+        with open(self.TestFile('bss_data'), 'rb') as fd:
             TestFunctional._MakeInputFile('-boot', fd.read())
         data = self._DoReadFile('096_elf.dts')
 
     def testElfStripg(self):
         """Basic test of ELF entries"""
         self._SetupSplElf()
-        with open(self.TestFile('bss_data')) as fd:
+        with open(self.TestFile('bss_data'), 'rb') as fd:
             TestFunctional._MakeInputFile('-boot', fd.read())
         data = self._DoReadFile('097_elf_strip.dts')
 
@@ -1771,7 +1777,7 @@ class TestFunctional(unittest.TestCase):
         # We should not get an inmage, but there should be a map file
         self.assertFalse(os.path.exists(tools.GetOutputFilename('image.bin')))
         self.assertTrue(os.path.exists(map_fname))
-        map_data = tools.ReadFile(map_fname)
+        map_data = tools.ReadFile(map_fname, binary=False)
         self.assertEqual('''ImagePos    Offset      Size  Name
 <none>    00000000  00000007  main-section
 <none>     00000000  00000004  u-boot
@@ -1797,9 +1803,12 @@ class TestFunctional(unittest.TestCase):
 0000002c    00000000  00000004  u-boot
 ''', map_data)
         self.assertEqual(data,
-                         4 * chr(0x26) + U_BOOT_DATA + 12 * chr(0x21) +
-                         4 * chr(0x26) + U_BOOT_DATA + 12 * chr(0x61) +
-                         4 * chr(0x26) + U_BOOT_DATA + 8 * chr(0x26))
+                         tools.GetBytes(0x26, 4) + U_BOOT_DATA +
+                             tools.GetBytes(0x21, 12) +
+                         tools.GetBytes(0x26, 4) + U_BOOT_DATA +
+                             tools.GetBytes(0x61, 12) +
+                         tools.GetBytes(0x26, 4) + U_BOOT_DATA +
+                             tools.GetBytes(0x26, 8))
 
 
 if __name__ == "__main__":

tools/binman/state.py

@@ -7,7 +7,6 @@
 
 import hashlib
 import re
-from sets import Set
 
 import os
 import tools
@@ -24,10 +23,10 @@ entry_args = {}
 use_fake_dtb = False
 
 # Set of all device tree files references by images
-fdt_set = Set()
+fdt_set = set()
 
 # Same as above, but excluding the main one
-fdt_subset = Set()
+fdt_subset = set()
 
 # The DTB which contains the full image information
 main_dtb = None
@@ -136,7 +135,7 @@ def Prepare(images, dtb):
     main_dtb = dtb
     fdt_files.clear()
     fdt_files['u-boot.dtb'] = dtb
-    fdt_subset = Set()
+    fdt_subset = set()
     if not use_fake_dtb:
         for image in images.values():
             fdt_subset.update(image.GetFdtSet())

tools/dtoc/dtb_platdata.py

@@ -17,6 +17,7 @@ import sys
 
 import fdt
 import fdt_util
+import tools
 
 # When we see these properties we ignore them - i.e. do not create a structure member
 PROP_IGNORE_LIST = [
@@ -99,7 +100,7 @@ def get_value(ftype, value):
     if ftype == fdt.TYPE_INT:
         return '%#x' % fdt_util.fdt32_to_cpu(value)
     elif ftype == fdt.TYPE_BYTE:
-        return '%#x' % ord(value[0])
+        return '%#x' % tools.ToByte(value[0])
     elif ftype == fdt.TYPE_STRING:
         return '"%s"' % value
     elif ftype == fdt.TYPE_BOOL:
@@ -449,7 +450,7 @@ class DtbPlatdata(object):
                 self.out(';\n')
             self.out('};\n')
 
-        for alias, struct_name in self._aliases.iteritems():
+        for alias, struct_name in self._aliases.items():
             if alias not in sorted(structs):
                 self.out('#define %s%s %s%s\n'% (STRUCT_PREFIX, alias,
                                                  STRUCT_PREFIX, struct_name))
@@ -464,7 +465,8 @@ class DtbPlatdata(object):
         var_name = conv_name_to_c(node.name)
         self.buf('static const struct %s%s %s%s = {\n' %
                  (STRUCT_PREFIX, struct_name, VAL_PREFIX, var_name))
-        for pname, prop in node.props.items():
+        for pname in sorted(node.props):
+            prop = node.props[pname]
             if pname in PROP_IGNORE_LIST or pname[0] == '#':
                 continue
             member_name = conv_name_to_c(prop.name)
@@ -498,7 +500,7 @@ class DtbPlatdata(object):
                         vals.append(get_value(prop.type, val))
 
                     # Put 8 values per line to avoid very long lines.
-                    for i in xrange(0, len(vals), 8):
+                    for i in range(0, len(vals), 8):
                         if i:
                             self.buf(',\n\t\t')
                         self.buf(', '.join(vals[i:i + 8]))

tools/dtoc/dtoc.py

@@ -25,6 +25,8 @@ options. For more information about the use of this options and tool please
 see doc/driver-model/of-plat.txt
 """
 
+from __future__ import print_function
+
 from optparse import OptionParser
 import os
 import sys
@@ -64,11 +66,11 @@ def run_tests(args):
             suite = unittest.TestLoader().loadTestsFromTestCase(module)
         suite.run(result)
 
-    print result
+    print(result)
     for _, err in result.errors:
-        print err
+        print(err)
     for _, err in result.failures:
-        print err
+        print(err)
 
 def RunTestCoverage():
     """Run the tests and check that we get 100% coverage"""

tools/dtoc/fdt.py

@@ -11,6 +11,7 @@ import sys
 import fdt_util
 import libfdt
 from libfdt import QUIET_NOTFOUND
+import tools
 
 # This deals with a device tree, presenting it as an assortment of Node and
 # Prop objects, representing nodes and properties, respectively. This file
@@ -28,6 +29,66 @@ def CheckErr(errnum, msg):
         raise ValueError('Error %d: %s: %s' %
             (errnum, libfdt.fdt_strerror(errnum), msg))
 
+
+def BytesToValue(data):
+    """Converts a string of bytes into a type and value
+
+    Args:
+        A bytes value (which on Python 2 is an alias for str)
+
+    Return:
+        A tuple:
+            Type of data
+            Data, either a single element or a list of elements. Each element
+            is one of:
+                TYPE_STRING: str/bytes value from the property
+                TYPE_INT: a byte-swapped integer stored as a 4-byte str/bytes
+                TYPE_BYTE: a byte stored as a single-byte str/bytes
+    """
+    data = bytes(data)
+    size = len(data)
+    strings = data.split(b'\0')
+    is_string = True
+    count = len(strings) - 1
+    if count > 0 and not len(strings[-1]):
+        for string in strings[:-1]:
+            if not string:
+                is_string = False
+                break
+            for ch in string:
+                # Handle Python 2 treating bytes as str
+                if type(ch) == str:
+                    ch = ord(ch)
+                if ch < 32 or ch > 127:
+                    is_string = False
+                    break
+    else:
+        is_string = False
+    if is_string:
+        if count == 1: 
+            if sys.version_info[0] >= 3:  # pragma: no cover
+                return TYPE_STRING, strings[0].decode()
+            else:
+                return TYPE_STRING, strings[0]
+        else:
+            if sys.version_info[0] >= 3:  # pragma: no cover
+                return TYPE_STRING, [s.decode() for s in strings[:-1]]
+            else:
+                return TYPE_STRING, strings[:-1]
+    if size % 4:
+        if size == 1:
+            return TYPE_BYTE, tools.ToChar(data[0])
+        else:
+            return TYPE_BYTE, [tools.ToChar(ch) for ch in list(data)]
+    val = []
+    for i in range(0, size, 4):
+        val.append(data[i:i + 4])
+    if size == 4:
+        return TYPE_INT, val[0]
+    else:
+        return TYPE_INT, val
+
+
 class Prop:
     """A device tree property
 
@@ -37,18 +98,18 @@ class Prop:
             bytes
         type: Value type
     """
-    def __init__(self, node, offset, name, bytes):
+    def __init__(self, node, offset, name, data):
         self._node = node
         self._offset = offset
         self.name = name
         self.value = None
-        self.bytes = str(bytes)
+        self.bytes = bytes(data)
         self.dirty = False
-        if not bytes:
+        if not data:
             self.type = TYPE_BOOL
             self.value = True
             return
-        self.type, self.value = self.BytesToValue(bytes)
+        self.type, self.value = BytesToValue(bytes(data))
 
     def RefreshOffset(self, poffset):
         self._offset = poffset
@@ -87,55 +148,6 @@ class Prop:
             while len(self.value) < len(newprop.value):
                 self.value.append(val)
 
-    def BytesToValue(self, bytes):
-        """Converts a string of bytes into a type and value
-
-        Args:
-            A string containing bytes
-
-        Return:
-            A tuple:
-                Type of data
-                Data, either a single element or a list of elements. Each element
-                is one of:
-                    TYPE_STRING: string value from the property
-                    TYPE_INT: a byte-swapped integer stored as a 4-byte string
-                    TYPE_BYTE: a byte stored as a single-byte string
-        """
-        bytes = str(bytes)
-        size = len(bytes)
-        strings = bytes.split('\0')
-        is_string = True
-        count = len(strings) - 1
-        if count > 0 and not strings[-1]:
-            for string in strings[:-1]:
-                if not string:
-                    is_string = False
-                    break
-                for ch in string:
-                    if ch < ' ' or ch > '~':
-                        is_string = False
-                        break
-        else:
-            is_string = False
-        if is_string:
-            if count == 1:
-                return TYPE_STRING, strings[0]
-            else:
-                return TYPE_STRING, strings[:-1]
-        if size % 4:
-            if size == 1:
-                return TYPE_BYTE, bytes[0]
-            else:
-                return TYPE_BYTE, list(bytes)
-        val = []
-        for i in range(0, size, 4):
-            val.append(bytes[i:i + 4])
-        if size == 4:
-            return TYPE_INT, val[0]
-        else:
-            return TYPE_INT, val
-
     @classmethod
     def GetEmpty(self, type):
         """Get an empty / zero value of the given type
@@ -181,8 +193,8 @@ class Prop:
         Args:
             bytes: New property value to set
         """
-        self.bytes = str(bytes)
-        self.type, self.value = self.BytesToValue(bytes)
+        self.bytes = bytes
+        self.type, self.value = BytesToValue(bytes)
         self.dirty = True
 
     def Sync(self, auto_resize=False):
@@ -334,7 +346,8 @@ class Node:
         Args:
             prop_name: Name of property
         """
-        self.props[prop_name] = Prop(self, None, prop_name, '\0' * 4)
+        self.props[prop_name] = Prop(self, None, prop_name,
+                                     tools.GetBytes(0, 4))
 
     def AddEmptyProp(self, prop_name, len):
         """Add a property with a fixed data size, for filling in later
@@ -346,7 +359,7 @@ class Node:
             prop_name: Name of property
             len: Length of data in property
         """
-        value = chr(0) * len
+        value = tools.GetBytes(0, len)
         self.props[prop_name] = Prop(self, None, prop_name, value)
 
     def SetInt(self, prop_name, val):
@@ -385,7 +398,9 @@ class Node:
             prop_name: Name of property to set
             val: String value to set (will be \0-terminated in DT)
         """
-        self.props[prop_name].SetData(val + chr(0))
+        if sys.version_info[0] >= 3:  # pragma: no cover
+            val = bytes(val, 'utf-8')
+        self.props[prop_name].SetData(val + b'\0')
 
     def AddString(self, prop_name, val):
         """Add a new string property to a node
@@ -397,7 +412,9 @@ class Node:
             prop_name: Name of property to add
             val: String value of property
         """
-        self.props[prop_name] = Prop(self, None, prop_name, val + chr(0))
+        if sys.version_info[0] >= 3:  # pragma: no cover
+            val = bytes(val, 'utf-8')
+        self.props[prop_name] = Prop(self, None, prop_name, val + b'\0')
 
     def AddSubnode(self, name):
         """Add a new subnode to the node
@@ -448,8 +465,11 @@ class Node:
 
         # Sync properties now, whose offsets should not have been disturbed.
         # We do this after subnodes, since this disturbs the offsets of these
-        # properties.
-        prop_list = sorted(self.props.values(), key=lambda prop: prop._offset,
+        # properties. Note that new properties will have an offset of None here,
+        # which Python 3 cannot sort against int. So use a large value instead
+        # to ensure that the new properties are added first.
+        prop_list = sorted(self.props.values(),
+                           key=lambda prop: prop._offset or 1 << 31,
                            reverse=True)
         for prop in prop_list:
             prop.Sync(auto_resize)
@@ -469,7 +489,7 @@ class Fdt:
         if self._fname:
             self._fname = fdt_util.EnsureCompiled(self._fname)
 
-            with open(self._fname) as fd:
+            with open(self._fname, 'rb') as fd:
                 self._fdt_obj = libfdt.Fdt(fd.read())
 
     @staticmethod
@@ -483,7 +503,7 @@ class Fdt:
             Fdt object containing the data
         """
         fdt = Fdt(None)
-        fdt._fdt_obj = libfdt.Fdt(bytearray(data))
+        fdt._fdt_obj = libfdt.Fdt(bytes(data))
         return fdt
 
     def LookupPhandle(self, phandle):
@@ -573,7 +593,7 @@ class Fdt:
         Returns:
             The FDT contents as a string of bytes
         """
-        return self._fdt_obj.as_bytearray()
+        return bytes(self._fdt_obj.as_bytearray())
 
     def GetFdtObj(self):
         """Get the contents of the FDT

tools/dtoc/fdt_util.py

@@ -16,14 +16,6 @@ import tempfile
 import command
 import tools
 
-VERSION3 = sys.version_info > (3, 0)
-
-def get_plain_bytes(val):
-    """Handle Python 3 strings"""
-    if isinstance(val, bytes):
-        val = val.decode('utf-8')
-    return val.encode('raw_unicode_escape')
-
 def fdt32_to_cpu(val):
     """Convert a device tree cell to an integer
 
@@ -33,9 +25,6 @@ def fdt32_to_cpu(val):
     Return:
         A native-endian integer value
     """
-    if VERSION3:
-        # This code is not reached in Python 2
-        val = get_plain_bytes(val)  # pragma: no cover
     return struct.unpack('>I', val)[0]
 
 def fdt_cells_to_cpu(val, cells):
@@ -45,11 +34,11 @@ def fdt_cells_to_cpu(val, cells):
         Value to convert (array of one or more 4-character strings)
 
     Return:
-        A native-endian long value
+        A native-endian integer value
     """
     if not cells:
         return 0
-    out = long(fdt32_to_cpu(val[0]))
+    out = int(fdt32_to_cpu(val[0]))
     if cells == 2:
         out = out << 32 | fdt32_to_cpu(val[1])
     return out

tools/dtoc/test_dtoc.py

@@ -8,6 +8,8 @@ This includes unit tests for some functions and functional tests for the dtoc
 tool.
 """
 
+from __future__ import print_function
+
 import collections
 import os
 import struct
@@ -97,7 +99,7 @@ class TestDtoc(unittest.TestCase):
         if expected != actual:
             self._WritePythonString('/tmp/binman.expected', expected)
             self._WritePythonString('/tmp/binman.actual', actual)
-            print 'Failures written to /tmp/binman.{expected,actual}'
+            print('Failures written to /tmp/binman.{expected,actual}')
         self.assertEquals(expected, actual)
 
     def test_name(self):
@@ -197,16 +199,16 @@ struct dtd_sandbox_spl_test_2 {
             data = infile.read()
         self._CheckStrings(C_HEADER + '''
 static const struct dtd_sandbox_spl_test dtv_spl_test = {
+\t.boolval\t\t= true,
 \t.bytearray\t\t= {0x6, 0x0, 0x0},
 \t.byteval\t\t= 0x5,
+\t.intarray\t\t= {0x2, 0x3, 0x4, 0x0},
 \t.intval\t\t\t= 0x1,
-\t.notstring\t\t= {0x20, 0x21, 0x22, 0x10, 0x0},
 \t.longbytearray\t\t= {0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10,
 \t\t0x11},
-\t.stringval\t\t= "message",
-\t.boolval\t\t= true,
-\t.intarray\t\t= {0x2, 0x3, 0x4, 0x0},
+\t.notstring\t\t= {0x20, 0x21, 0x22, 0x10, 0x0},
 \t.stringarray\t\t= {"multi-word", "message", ""},
+\t.stringval\t\t= "message",
 };
 U_BOOT_DEVICE(spl_test) = {
 \t.name\t\t= "sandbox_spl_test",
@@ -217,12 +219,12 @@ U_BOOT_DEVICE(spl_test) = {
 static const struct dtd_sandbox_spl_test dtv_spl_test2 = {
 \t.bytearray\t\t= {0x1, 0x23, 0x34},
 \t.byteval\t\t= 0x8,
+\t.intarray\t\t= {0x5, 0x0, 0x0, 0x0},
 \t.intval\t\t\t= 0x3,
 \t.longbytearray\t\t= {0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
 \t\t0x0},
-\t.stringval\t\t= "message2",
-\t.intarray\t\t= {0x5, 0x0, 0x0, 0x0},
 \t.stringarray\t\t= {"another", "multi-word", "message"},
+\t.stringval\t\t= "message2",
 };
 U_BOOT_DEVICE(spl_test2) = {
 \t.name\t\t= "sandbox_spl_test",

tools/dtoc/test_fdt.py

@@ -4,6 +4,8 @@
 # Written by Simon Glass <sjg@chromium.org>
 #
 
+from __future__ import print_function
+
 from optparse import OptionParser
 import glob
 import os
@@ -17,7 +19,7 @@ for dirname in ['../patman', '..']:
 
 import command
 import fdt
-from fdt import TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL
+from fdt import TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL, BytesToValue
 import fdt_util
 from fdt_util import fdt32_to_cpu
 import libfdt
@@ -45,7 +47,7 @@ def _GetPropertyValue(dtb, node, prop_name):
     # Add 12, which is sizeof(struct fdt_property), to get to start of data
     offset = prop.GetOffset() + 12
     data = dtb.GetContents()[offset:offset + len(prop.value)]
-    return prop, [chr(x) for x in data]
+    return prop, [tools.ToChar(x) for x in data]
 
 
 class TestFdt(unittest.TestCase):
@@ -83,13 +85,13 @@ class TestFdt(unittest.TestCase):
     def testFlush(self):
         """Check that we can flush the device tree out to its file"""
         fname = self.dtb._fname
-        with open(fname) as fd:
+        with open(fname, 'rb') as fd:
             data = fd.read()
         os.remove(fname)
         with self.assertRaises(IOError):
-            open(fname)
+            open(fname, 'rb')
         self.dtb.Flush()
-        with open(fname) as fd:
+        with open(fname, 'rb') as fd:
             data = fd.read()
 
     def testPack(self):
@@ -119,6 +121,10 @@ class TestFdt(unittest.TestCase):
         node = self.dtb.GetNode('/spl-test')
         self.assertEqual(self.dtb, node.GetFdt())
 
+    def testBytesToValue(self):
+        self.assertEqual(BytesToValue(b'this\0is\0'),
+                         (TYPE_STRING, ['this', 'is']))
+
 class TestNode(unittest.TestCase):
     """Test operation of the Node class"""
 
@@ -277,7 +283,7 @@ class TestProp(unittest.TestCase):
         """Tests the GetEmpty() function for the various supported types"""
         self.assertEqual(True, fdt.Prop.GetEmpty(fdt.TYPE_BOOL))
         self.assertEqual(chr(0), fdt.Prop.GetEmpty(fdt.TYPE_BYTE))
-        self.assertEqual(chr(0) * 4, fdt.Prop.GetEmpty(fdt.TYPE_INT))
+        self.assertEqual(tools.GetBytes(0, 4), fdt.Prop.GetEmpty(fdt.TYPE_INT))
         self.assertEqual('', fdt.Prop.GetEmpty(fdt.TYPE_STRING))
 
     def testGetOffset(self):
@@ -381,7 +387,7 @@ class TestProp(unittest.TestCase):
         self.node.AddString('string', val)
         self.dtb.Sync(auto_resize=True)
         data = self.fdt.getprop(self.node.Offset(), 'string')
-        self.assertEqual(val + '\0', data)
+        self.assertEqual(tools.ToBytes(val) + b'\0', data)
 
         self.fdt.pack()
         self.node.SetString('string', val + 'x')
@@ -391,21 +397,21 @@ class TestProp(unittest.TestCase):
         self.node.SetString('string', val[:-1])
 
         prop = self.node.props['string']
-        prop.SetData(val)
+        prop.SetData(tools.ToBytes(val))
         self.dtb.Sync(auto_resize=False)
         data = self.fdt.getprop(self.node.Offset(), 'string')
-        self.assertEqual(val, data)
+        self.assertEqual(tools.ToBytes(val), data)
 
         self.node.AddEmptyProp('empty', 5)
         self.dtb.Sync(auto_resize=True)
         prop = self.node.props['empty']
-        prop.SetData(val)
+        prop.SetData(tools.ToBytes(val))
         self.dtb.Sync(auto_resize=False)
         data = self.fdt.getprop(self.node.Offset(), 'empty')
-        self.assertEqual(val, data)
+        self.assertEqual(tools.ToBytes(val), data)
 
-        self.node.SetData('empty', '123')
-        self.assertEqual('123', prop.bytes)
+        self.node.SetData('empty', b'123')
+        self.assertEqual(b'123', prop.bytes)
 
     def testFromData(self):
         dtb2 = fdt.Fdt.FromData(self.dtb.GetContents())
@@ -496,18 +502,22 @@ class TestFdtUtil(unittest.TestCase):
         self.assertEqual(2, fdt_util.fdt_cells_to_cpu(val, 1))
 
         dtb2 = fdt.FdtScan('tools/dtoc/dtoc_test_addr64.dts')
-        node2 = dtb2.GetNode('/test1')
-        val = node2.props['reg'].value
+        node1 = dtb2.GetNode('/test1')
+        val = node1.props['reg'].value
         self.assertEqual(0x1234, fdt_util.fdt_cells_to_cpu(val, 2))
 
+        node2 = dtb2.GetNode('/test2')
+        val = node2.props['reg'].value
+        self.assertEqual(0x1234567890123456, fdt_util.fdt_cells_to_cpu(val, 2))
+        self.assertEqual(0x9876543210987654, fdt_util.fdt_cells_to_cpu(val[2:],
+                                                                       2))
+        self.assertEqual(0x12345678, fdt_util.fdt_cells_to_cpu(val, 1))
+
     def testEnsureCompiled(self):
         """Test a degenerate case of this function"""
         dtb = fdt_util.EnsureCompiled('tools/dtoc/dtoc_test_simple.dts')
         self.assertEqual(dtb, fdt_util.EnsureCompiled(dtb))
 
-    def testGetPlainBytes(self):
-        self.assertEqual('fred', fdt_util.get_plain_bytes('fred'))
-
 
 def RunTestCoverage():
     """Run the tests and check that we get 100% coverage"""
@@ -535,11 +545,11 @@ def RunTests(args):
             suite = unittest.TestLoader().loadTestsFromTestCase(module)
         suite.run(result)
 
-    print result
+    print(result)
     for _, err in result.errors:
-        print err
+        print(err)
     for _, err in result.failures:
-        print err
+        print(err)
 
 if __name__ != '__main__':
     sys.exit(1)

tools/patman/cros_subprocess.py

@@ -54,7 +54,7 @@ class Popen(subprocess.Popen):
     """
 
     def __init__(self, args, stdin=None, stdout=PIPE_PTY, stderr=PIPE_PTY,
-                 shell=False, cwd=None, env=None, **kwargs):
+                 shell=False, cwd=None, env=None, binary=False, **kwargs):
         """Cut-down constructor
 
         Args:
@@ -72,6 +72,7 @@ class Popen(subprocess.Popen):
         """
         stdout_pty = None
         stderr_pty = None
+        self.binary = binary
 
         if stdout == PIPE_PTY:
             stdout_pty = pty.openpty()
@@ -100,6 +101,19 @@ class Popen(subprocess.Popen):
         if kwargs:
             raise ValueError("Unit tests do not test extra args - please add tests")
 
+    def ConvertData(self, data):
+        """Convert stdout/stderr data to the correct format for output
+
+        Args:
+            data: Data to convert, or None for ''
+
+        Returns:
+            Converted data, as bytes
+        """
+        if data is None:
+            return b''
+        return data
+
     def CommunicateFilter(self, output):
         """Interact with process: Read data from stdout and stderr.
 
@@ -156,11 +170,11 @@ class Popen(subprocess.Popen):
                 self.stdin.close()
         if self.stdout:
             read_set.append(self.stdout)
-            stdout = []
+            stdout = b''
         if self.stderr and self.stderr != self.stdout:
             read_set.append(self.stderr)
-            stderr = []
-        combined = []
+            stderr = b''
+        combined = b''
 
         input_offset = 0
         while read_set or write_set:
@@ -186,46 +200,40 @@ class Popen(subprocess.Popen):
                     write_set.remove(self.stdin)
 
             if self.stdout in rlist:
-                data = ""
+                data = b''
                 # We will get an error on read if the pty is closed
                 try:
                     data = os.read(self.stdout.fileno(), 1024)
                 except OSError:
                     pass
-                if data == "":
+                if not len(data):
                     self.stdout.close()
                     read_set.remove(self.stdout)
                 else:
-                    stdout.append(data)
-                    combined.append(data)
+                    stdout += data
+                    combined += data
                     if output:
                         output(sys.stdout, data)
             if self.stderr in rlist:
-                data = ""
+                data = b''
                 # We will get an error on read if the pty is closed
                 try:
                     data = os.read(self.stderr.fileno(), 1024)
                 except OSError:
                     pass
-                if data == "":
+                if not len(data):
                     self.stderr.close()
                     read_set.remove(self.stderr)
                 else:
-                    stderr.append(data)
-                    combined.append(data)
+                    stderr += data
+                    combined += data
                     if output:
                         output(sys.stderr, data)
 
         # All data exchanged.    Translate lists into strings.
-        if stdout is not None:
-            stdout = ''.join(stdout)
-        else:
-            stdout = ''
-        if stderr is not None:
-            stderr = ''.join(stderr)
-        else:
-            stderr = ''
-        combined = ''.join(combined)
+        stdout = self.ConvertData(stdout)
+        stderr = self.ConvertData(stderr)
+        combined = self.ConvertData(combined)
 
         # Translate newlines, if requested.    We cannot let the file
         # object do the translation: It is based on stdio, which is

tools/patman/func_test.py

@@ -12,15 +12,20 @@ import sys
 import tempfile
 import unittest
 
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 import gitutil
 import patchstream
 import settings
+import tools
 
 
 @contextlib.contextmanager
 def capture():
     import sys
-    from cStringIO import StringIO
     oldout,olderr = sys.stdout, sys.stderr
     try:
         out=[StringIO(), StringIO()]
@@ -124,10 +129,10 @@ class TestFunctional(unittest.TestCase):
         """
         process_tags = True
         ignore_bad_tags = True
-        stefan = u'Stefan Brüns <stefan.bruens@rwth-aachen.de>'
+        stefan = b'Stefan Br\xc3\xbcns <stefan.bruens@rwth-aachen.de>'.decode('utf-8')
         rick = 'Richard III <richard@palace.gov>'
-        mel = u'Lord Mëlchett <clergy@palace.gov>'
-        ed = u'Lond Edmund Blackaddër <weasel@blackadder.org'
+        mel = b'Lord M\xc3\xablchett <clergy@palace.gov>'.decode('utf-8')
+        ed = b'Lond Edmund Blackadd\xc3\xabr <weasel@blackadder.org'.decode('utf-8')
         fred = 'Fred Bloggs <f.bloggs@napier.net>'
         add_maintainers = [stefan, rick]
         dry_run = True
@@ -159,7 +164,6 @@ class TestFunctional(unittest.TestCase):
         os.remove(cc_file)
 
         lines = out[0].splitlines()
-        #print '\n'.join(lines)
         self.assertEqual('Cleaned %s patches' % len(series.commits), lines[0])
         self.assertEqual('Change log missing for v2', lines[1])
         self.assertEqual('Change log missing for v3', lines[2])
@@ -174,27 +178,30 @@ class TestFunctional(unittest.TestCase):
             while 'Cc:' in lines[line]:
                 line += 1
         self.assertEqual('To:	  u-boot@lists.denx.de', lines[line])
-        self.assertEqual('Cc:	  %s' % stefan.encode('utf-8'), lines[line + 1])
+        self.assertEqual('Cc:	  %s' % tools.FromUnicode(stefan),
+                         lines[line + 1])
         self.assertEqual('Version:  3', lines[line + 2])
         self.assertEqual('Prefix:\t  RFC', lines[line + 3])
         self.assertEqual('Cover: 4 lines', lines[line + 4])
         line += 5
-        self.assertEqual('      Cc:  %s' % mel.encode('utf-8'), lines[line + 0])
-        self.assertEqual('      Cc:  %s' % rick, lines[line + 1])
-        self.assertEqual('      Cc:  %s' % fred, lines[line + 2])
-        self.assertEqual('      Cc:  %s' % ed.encode('utf-8'), lines[line + 3])
+        self.assertEqual('      Cc:  %s' % fred, lines[line + 0])
+        self.assertEqual('      Cc:  %s' % tools.FromUnicode(ed),
+                         lines[line + 1])
+        self.assertEqual('      Cc:  %s' % tools.FromUnicode(mel),
+                         lines[line + 2])
+        self.assertEqual('      Cc:  %s' % rick, lines[line + 3])
         expected = ('Git command: git send-email --annotate '
                     '--in-reply-to="%s" --to "u-boot@lists.denx.de" '
                     '--cc "%s" --cc-cmd "%s --cc-cmd %s" %s %s'
                     % (in_reply_to, stefan, sys.argv[0], cc_file, cover_fname,
-                       ' '.join(args))).encode('utf-8')
+                       ' '.join(args)))
         line += 4
-        self.assertEqual(expected, lines[line])
+        self.assertEqual(expected, tools.ToUnicode(lines[line]))
 
-        self.assertEqual(('%s %s, %s' % (args[0], rick, stefan))
-                         .encode('utf-8'), cc_lines[0])
-        self.assertEqual(('%s %s, %s, %s, %s' % (args[1], fred, rick, stefan,
-                                            ed)).encode('utf-8'), cc_lines[1])
+        self.assertEqual(('%s %s, %s' % (args[0], rick, stefan)),
+                         tools.ToUnicode(cc_lines[0]))
+        self.assertEqual(('%s %s, %s, %s, %s' % (args[1], fred, ed, rick,
+                                     stefan)), tools.ToUnicode(cc_lines[1]))
 
         expected = '''
 This is a test of how the cover
@@ -223,7 +230,6 @@ Simon Glass (2):
 
 '''
         lines = open(cover_fname).read().splitlines()
-        #print '\n'.join(lines)
         self.assertEqual(
                 'Subject: [RFC PATCH v3 0/2] test: A test patch series',
                 lines[3])
@@ -231,7 +237,6 @@ Simon Glass (2):
 
         for i, fname in enumerate(args):
             lines = open(fname).read().splitlines()
-            #print '\n'.join(lines)
             subject = [line for line in lines if line.startswith('Subject')]
             self.assertEqual('Subject: [RFC %d/%d]' % (i + 1, count),
                              subject[0][:18])

tools/patman/gitutil.py

@@ -12,6 +12,7 @@ import terminal
 
 import checkpatch
 import settings
+import tools
 
 # True to use --no-decorate - we check this in Setup()
 use_no_decorate = True
@@ -325,6 +326,7 @@ def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True):
         raw += LookupEmail(item, alias, raise_on_error=raise_on_error)
     result = []
     for item in raw:
+        item = tools.FromUnicode(item)
         if not item in result:
             result.append(item)
     if tag:
@@ -395,7 +397,7 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname,
         git_config_to = command.Output('git', 'config', 'sendemail.to',
                                        raise_on_error=False)
         if not git_config_to:
-            print ("No recipient.\n"
+            print("No recipient.\n"
                   "Please add something like this to a commit\n"
                   "Series-to: Fred Bloggs <f.blogs@napier.co.nz>\n"
                   "Or do something like this\n"
@@ -410,9 +412,7 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname,
     if smtp_server:
         cmd.append('--smtp-server=%s' % smtp_server)
     if in_reply_to:
-        if type(in_reply_to) != str:
-            in_reply_to = in_reply_to.encode('utf-8')
-        cmd.append('--in-reply-to="%s"' % in_reply_to)
+        cmd.append('--in-reply-to="%s"' % tools.FromUnicode(in_reply_to))
     if thread:
         cmd.append('--thread')
 

tools/patman/patman.py

-#!/usr/bin/env python2
+#!/usr/bin/env python
 # SPDX-License-Identifier: GPL-2.0+
 #
 # Copyright (c) 2011 The Chromium OS Authors.

tools/patman/series.py

@@ -11,6 +11,7 @@ import get_maintainer
 import gitutil
 import settings
 import terminal
+import tools
 
 # Series-xxx tags that we understand
 valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name',
@@ -114,16 +115,16 @@ class Series(dict):
             commit = self.commits[upto]
             print(col.Color(col.GREEN, '   %s' % args[upto]))
             cc_list = list(self._generated_cc[commit.patch])
-            for email in set(cc_list) - to_set - cc_set:
+            for email in sorted(set(cc_list) - to_set - cc_set):
                 if email == None:
                     email = col.Color(col.YELLOW, "<alias '%s' not found>"
                             % tag)
                 if email:
                     print('      Cc: ', email)
         print
-        for item in to_set:
+        for item in sorted(to_set):
             print('To:\t ', item)
-        for item in cc_set - to_set:
+        for item in sorted(cc_set - to_set):
             print('Cc:\t ', item)
         print('Version: ', self.get('version'))
         print('Prefix:\t ', self.get('prefix'))
@@ -131,7 +132,7 @@ class Series(dict):
             print('Cover: %d lines' % len(self.cover))
             cover_cc = gitutil.BuildEmailList(self.get('cover_cc', ''))
             all_ccs = itertools.chain(cover_cc, *self._generated_cc.values())
-            for email in set(all_ccs) - to_set - cc_set:
+            for email in sorted(set(all_ccs) - to_set - cc_set):
                     print('      Cc: ', email)
         if cmd:
             print('Git command: %s' % cmd)
@@ -238,19 +239,18 @@ class Series(dict):
             for x in set(cc) & set(settings.bounces):
                 print(col.Color(col.YELLOW, 'Skipping "%s"' % x))
             cc = set(cc) - set(settings.bounces)
-            cc = [m.encode('utf-8') if type(m) != str else m for m in cc]
+            cc = [tools.FromUnicode(m) for m in cc]
             if limit is not None:
                 cc = cc[:limit]
             all_ccs += cc
-            print(commit.patch, ', '.join(set(cc)), file=fd)
+            print(commit.patch, ', '.join(sorted(set(cc))), file=fd)
             self._generated_cc[commit.patch] = cc
 
         if cover_fname:
             cover_cc = gitutil.BuildEmailList(self.get('cover_cc', ''))
-            cover_cc = [m.encode('utf-8') if type(m) != str else m
-                        for m in cover_cc]
-            cc_list = ', '.join([x.decode('utf-8')
-                                 for x in set(cover_cc + all_ccs)])
+            cover_cc = [tools.FromUnicode(m) for m in cover_cc]
+            cc_list = ', '.join([tools.ToUnicode(x)
+                                 for x in sorted(set(cover_cc + all_ccs))])
             print(cover_fname, cc_list.encode('utf-8'), file=fd)
 
         fd.close()

tools/patman/settings.py

@@ -14,6 +14,7 @@ import re
 
 import command
 import gitutil
+import tools
 
 """Default settings per-project.
 
@@ -57,26 +58,26 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser):
     # Check to make sure that bogus project gets general alias.
     >>> config = _ProjectConfigParser("zzz")
     >>> config.readfp(StringIO(sample_config))
-    >>> config.get("alias", "enemies")
-    u'Evil <evil@example.com>'
+    >>> str(config.get("alias", "enemies"))
+    'Evil <evil@example.com>'
 
     # Check to make sure that alias gets overridden by project.
     >>> config = _ProjectConfigParser("sm")
     >>> config.readfp(StringIO(sample_config))
-    >>> config.get("alias", "enemies")
-    u'Green G. <ugly@example.com>'
+    >>> str(config.get("alias", "enemies"))
+    'Green G. <ugly@example.com>'
 
     # Check to make sure that settings get merged with project.
     >>> config = _ProjectConfigParser("linux")
     >>> config.readfp(StringIO(sample_config))
-    >>> sorted(config.items("settings"))
-    [(u'am_hero', u'True'), (u'process_tags', u'False')]
+    >>> sorted((str(a), str(b)) for (a, b) in config.items("settings"))
+    [('am_hero', 'True'), ('process_tags', 'False')]
 
     # Check to make sure that settings works with unknown project.
     >>> config = _ProjectConfigParser("unknown")
     >>> config.readfp(StringIO(sample_config))
-    >>> sorted(config.items("settings"))
-    [(u'am_hero', u'True')]
+    >>> sorted((str(a), str(b)) for (a, b) in config.items("settings"))
+    [('am_hero', 'True')]
     """
     def __init__(self, project_name):
         """Construct _ProjectConfigParser.
@@ -99,17 +100,6 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser):
         for setting_name, setting_value in project_defaults.items():
             self.set(project_settings, setting_name, setting_value)
 
-    def _to_unicode(self, val):
-        """Make sure a value is of type 'unicode'
-
-        Args:
-            val: string or unicode object
-
-        Returns:
-            unicode version of val
-        """
-        return val if isinstance(val, unicode) else val.decode('utf-8')
-
     def get(self, section, option, *args, **kwargs):
         """Extend SafeConfigParser to try project_section before section.
 
@@ -127,7 +117,7 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser):
             val = ConfigParser.SafeConfigParser.get(
                 self, section, option, *args, **kwargs
             )
-        return self._to_unicode(val)
+        return tools.ToUnicode(val)
 
     def items(self, section, *args, **kwargs):
         """Extend SafeConfigParser to add project_section to section.
@@ -162,8 +152,8 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser):
 
         item_dict = dict(top_items)
         item_dict.update(project_items)
-        return {(self._to_unicode(item), self._to_unicode(val))
-                for item, val in item_dict.iteritems()}
+        return {(tools.ToUnicode(item), tools.ToUnicode(val))
+                for item, val in item_dict.items()}
 
 def ReadGitAliases(fname):
     """Read a git alias file. This is in the form used by git:

tools/patman/test_util.py

@@ -3,6 +3,8 @@
 # Copyright (c) 2016 Google, Inc
 #
 
+from __future__ import print_function
+
 from contextlib import contextmanager
 import glob
 import os
@@ -15,6 +17,8 @@ try:
 except ImportError:
   from io import StringIO
 
+PYTHON = 'python%d' % sys.version_info[0]
+
 
 def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None):
     """Run tests and check that we get 100% coverage
@@ -41,11 +45,12 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None):
     else:
         glob_list = []
     glob_list += exclude_list
-    glob_list += ['*libfdt.py', '*site-packages*']
-    cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools python-coverage run '
-           '--omit "%s" %s -P1 -t' % (build_dir, ','.join(glob_list), prog))
+    glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*']
+    cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools %s-coverage run '
+           '--omit "%s" %s -P1 -t' % (build_dir, PYTHON, ','.join(glob_list),
+                                      prog))
     os.system(cmd)
-    stdout = command.Output('python-coverage', 'report')
+    stdout = command.Output('%s-coverage' % PYTHON, 'report')
     lines = stdout.splitlines()
     if required:
         # Convert '/path/to/name.py' just the module name 'name'
@@ -54,18 +59,18 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None):
         missing_list = required
         missing_list.difference_update(test_set)
         if missing_list:
-            print 'Missing tests for %s' % (', '.join(missing_list))
-            print stdout
+            print('Missing tests for %s' % (', '.join(missing_list)))
+            print(stdout)
             ok = False
 
     coverage = lines[-1].split(' ')[-1]
     ok = True
-    print coverage
+    print(coverage)
     if coverage != '100%':
-        print stdout
-        print ("Type 'python-coverage html' to get a report in "
-               'htmlcov/index.html')
-        print 'Coverage error: %s, but should be 100%%' % coverage
+        print(stdout)
+        print("Type '%s-coverage html' to get a report in "
+              'htmlcov/index.html' % PYTHON)
+        print('Coverage error: %s, but should be 100%%' % coverage)
         ok = False
     if not ok:
         raise ValueError('Test coverage failure')

tools/patman/tools.py

@@ -7,6 +7,7 @@ import command
 import glob
 import os
 import shutil
+import sys
 import tempfile
 
 import tout
@@ -167,9 +168,9 @@ def PathHasFile(fname):
             return True
     return False
 
-def Run(name, *args):
+def Run(name, *args, **kwargs):
     try:
-        return command.Run(name, *args, cwd=outdir, capture=True)
+        return command.Run(name, *args, cwd=outdir, capture=True, **kwargs)
     except:
         if not PathHasFile(name):
             msg = "Plesae install tool '%s'" % name
@@ -213,7 +214,7 @@ def Filename(fname):
     # If not found, just return the standard, unchanged path
     return fname
 
-def ReadFile(fname):
+def ReadFile(fname, binary=True):
     """Read and return the contents of a file.
 
     Args:
@@ -222,7 +223,7 @@ def ReadFile(fname):
     Returns:
       data read from file, as a string.
     """
-    with open(Filename(fname), 'rb') as fd:
+    with open(Filename(fname), binary and 'rb' or 'r') as fd:
         data = fd.read()
     #self._out.Info("Read file '%s' size %d (%#0x)" %
                    #(fname, len(data), len(data)))
@@ -239,3 +240,105 @@ def WriteFile(fname, data):
                    #(fname, len(data), len(data)))
     with open(Filename(fname), 'wb') as fd:
         fd.write(data)
+
+def GetBytes(byte, size):
+    """Get a string of bytes of a given size
+
+    This handles the unfortunate different between Python 2 and Python 2.
+
+    Args:
+        byte: Numeric byte value to use
+        size: Size of bytes/string to return
+
+    Returns:
+        A bytes type with 'byte' repeated 'size' times
+    """
+    if sys.version_info[0] >= 3:
+        data = bytes([byte]) * size
+    else:
+        data = chr(byte) * size
+    return data
+
+def ToUnicode(val):
+    """Make sure a value is a unicode string
+
+    This allows some amount of compatibility between Python 2 and Python3. For
+    the former, it returns a unicode object.
+
+    Args:
+        val: string or unicode object
+
+    Returns:
+        unicode version of val
+    """
+    if sys.version_info[0] >= 3:
+        return val
+    return val if isinstance(val, unicode) else val.decode('utf-8')
+
+def FromUnicode(val):
+    """Make sure a value is a non-unicode string
+
+    This allows some amount of compatibility between Python 2 and Python3. For
+    the former, it converts a unicode object to a string.
+
+    Args:
+        val: string or unicode object
+
+    Returns:
+        non-unicode version of val
+    """
+    if sys.version_info[0] >= 3:
+        return val
+    return val if isinstance(val, str) else val.encode('utf-8')
+
+def ToByte(ch):
+    """Convert a character to an ASCII value
+
+    This is useful because in Python 2 bytes is an alias for str, but in
+    Python 3 they are separate types. This function converts the argument to
+    an ASCII value in either case.
+
+    Args:
+        ch: A string (Python 2) or byte (Python 3) value
+
+    Returns:
+        integer ASCII value for ch
+    """
+    return ord(ch) if type(ch) == str else ch
+
+def ToChar(byte):
+    """Convert a byte to a character
+
+    This is useful because in Python 2 bytes is an alias for str, but in
+    Python 3 they are separate types. This function converts an ASCII value to
+    a value with the appropriate type in either case.
+
+    Args:
+        byte: A byte or str value
+    """
+    return chr(byte) if type(byte) != str else byte
+
+def ToChars(byte_list):
+    """Convert a list of bytes to a str/bytes type
+
+    Args:
+        byte_list: List of ASCII values representing the string
+
+    Returns:
+        string made by concatenating all the ASCII values
+    """
+    return ''.join([chr(byte) for byte in byte_list])
+
+def ToBytes(string):
+    """Convert a str type into a bytes type
+
+    Args:
+        string: string to convert value
+
+    Returns:
+        Python 3: A bytes type
+        Python 2: A string type
+    """
+    if sys.version_info[0] >= 3:
+        return string.encode('utf-8')
+    return string

tools/proftool.c

@@ -205,12 +205,12 @@ static struct func_info *find_caller_by_offset(uint32_t offset)
 	return low >= 0 ? &func_list[low] : NULL;
 }
 
-static int read_calls(FILE *fin, int count)
+static int read_calls(FILE *fin, size_t count)
 {
 	struct trace_call *call_data;
 	int i;
 
-	notice("call count: %d\n", count);
+	notice("call count: %zu\n", count);
 	call_list = (struct trace_call *)calloc(count, sizeof(*call_data));
 	if (!call_list) {
 		error("Cannot allocate call_list\n");

tools/rmboard.py

+#! /usr/bin/python
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright 2019 Google LLC
+#
+
+"""
+Script to remove boards
+
+Usage:
+   rmboard.py <board_name>...
+
+A single commit is created for each board removed.
+
+Some boards may depend on files provided by another and this will cause
+problems, generally the removal of files which should not be removed.
+
+This script works by:
+    - Looking through the MAINTAINERS files which mention a board to find out
+        what files the board uses
+    - Looking through the Kconfig files which mention a board to find one that
+        needs to have material removed
+
+Search for ## to update the commit message manually.
+"""
+
+from __future__ import print_function
+
+import glob
+import os
+import re
+import sys
+
+# Bring in the patman libraries
+our_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(our_path, '../tools/patman'))
+
+import command
+
+def rm_kconfig_include(path):
+    """Remove a path from Kconfig files
+
+    This function finds the given path in a 'source' statement in a Kconfig
+    file and removes that line from the file. This is needed because the path
+    is going to be removed, so any reference to it will cause a problem with
+    Kconfig parsing.
+
+    The changes are made locally and then added to the git staging area.
+
+    Args:
+        path: Path to search for and remove
+    """
+    cmd = ['git', 'grep', path]
+    stdout = command.RunPipe([cmd], capture=True, raise_on_error=False).stdout
+    if not stdout:
+        return
+    fname = stdout.split(':')[0]
+
+    print("Fixing up '%s' to remove reference to '%s'" % (fname, path))
+    cmd = ['sed', '-i', '\|%s|d' % path, fname]
+    stdout = command.RunPipe([cmd], capture=True).stdout
+
+    cmd = ['git', 'add', fname]
+    stdout = command.RunPipe([cmd], capture=True).stdout
+
+def rm_board(board):
+    """Create a commit which removes a single board
+
+    This looks up the MAINTAINERS file to file files that need to be removed,
+    then removes pieces from the Kconfig files that mention the board.
+
+
+    Args:
+        board: Board name to remove
+    """
+
+    # Find all MAINTAINERS and Kconfig files which mention the board
+    cmd = ['git', 'grep', '-l', board]
+    stdout = command.RunPipe([cmd], capture=True).stdout
+    maintain = []
+    kconfig = []
+    for line in stdout.splitlines():
+        line = line.strip()
+        if 'MAINTAINERS' in line:
+            if line not in maintain:
+                maintain.append(line)
+        elif 'Kconfig' in line:
+            kconfig.append(line)
+    paths = []
+    cc = []
+
+    # Look through the MAINTAINERS file to find things to remove
+    for fname in maintain:
+        with open(fname) as fd:
+            for line in fd:
+                line = line.strip()
+                fields = re.split('[ \t]', line, 1)
+                if len(fields) == 2:
+                    if fields[0] == 'M:':
+                        cc.append(fields[1])
+                    elif fields[0] == 'F:':
+                        paths.append(fields[1].strip())
+
+    # Expand any wildcards in the MAINTAINERS file
+    real = []
+    for path in paths:
+        if path[-1] == '/':
+            path = path[:-1]
+        if '*' in path:
+            globbed = glob.glob(path)
+            print("Expanded '%s' to '%s'" % (path, globbed))
+            real += globbed
+        else:
+            real.append(path)
+
+    # Search for Kconfig files in the resulting list. Remove any 'source' lines
+    # which reference Kconfig files we want to remove
+    for path in real:
+        cmd = ['find', path]
+        stdout = (command.RunPipe([cmd], capture=True, raise_on_error=False).
+                  stdout)
+        for fname in stdout.splitlines():
+            if fname.endswith('Kconfig'):
+                rm_kconfig_include(fname)
+
+    # Remove unwanted files
+    cmd = ['git', 'rm', '-r'] + real
+    stdout = command.RunPipe([cmd], capture=True).stdout
+
+    ## Change the messages as needed
+    msg = '''arm: Remove %s board
+
+This board has not been converted to CONFIG_DM_MMC by the deadline.
+Remove it.
+
+''' % board
+    for name in cc:
+        msg += 'Patch-cc: %s\n' % name
+
+    # Create the commit
+    cmd = ['git', 'commit', '-s', '-m', msg]
+    stdout = command.RunPipe([cmd], capture=True).stdout
+
+    # Check if the board is mentioned anywhere else. The user will need to deal
+    # with this
+    cmd = ['git', 'grep', '-il', board]
+    print(command.RunPipe([cmd], capture=True, raise_on_error=False).stdout)
+    print(' '.join(cmd))
+
+for board in sys.argv[1:]:
+    rm_board(board)