Improve shrink transformer #8

booster-android-gradle-api/src/main/kotlin/com/didiglobal/booster/gradle/TransformInvocation.kt

@@ -5,10 +5,14 @@ import com.android.build.gradle.AppExtension
 import com.android.build.gradle.BaseExtension
 import com.android.build.gradle.LibraryExtension
 import com.android.build.gradle.api.BaseVariant
+import com.didiglobal.booster.kotlinx.execute
+import com.didiglobal.booster.kotlinx.head
+import com.didiglobal.booster.util.FileFinder
 import org.gradle.api.Project
 import org.gradle.api.internal.AbstractTask
+import org.objectweb.asm.ClassReader
+import org.objectweb.asm.tree.ClassNode
 import java.io.File
-import java.net.URLClassLoader
 
 /**
  * Represents the booster transform for
@@ -51,3 +55,32 @@ val TransformInvocation.compileClasspath: Collection<File>
  */
 val TransformInvocation.runtimeClasspath: Collection<File>
     get() = compileClasspath + project.getAndroid<BaseExtension>().bootClasspath
+
+/**
+ * Returns the application id
+ */
+val TransformInvocation.applicationId: String
+    get() {
+        val packages = variant.scope.symbolListWithPackageName.filter {
+            it.length() > 0
+        }.map {
+            it.head()!!
+        }.toSet()
+
+        return variant.scope.javac.map { classes ->
+            val base = classes.toURI()
+            FileFinder(classes) { file ->
+                file.name == "BuildConfig.class" && file.inputStream().use { bytecode ->
+                    ClassNode().also { klass ->
+                        ClassReader(bytecode).accept(klass, 0)
+                    }.fields.any {
+                        it.name == "APPLICATION_ID" && it.desc == "Ljava/lang/String;" && packages.contains(it.value)
+                    }
+                }
+            }.execute().map {
+                base.relativize(it.toURI()).path.let { path ->
+                    path.substring(0, path.lastIndexOf('/'))
+                }.replace('/', '.')
+            }.toSet()
+        }.flatten().single()
+    }

booster-gradle-base/src/main/kotlin/com/didiglobal/booster/util/FileFinder.kt → booster-android-gradle-api/src/main/kotlin/com/didiglobal/booster/util/FileFinder.kt

@@ -36,13 +36,4 @@ class FileFinder(private val roots: Collection<File>, private val filter: (File)
         return result + tasks.flatMap { it.join() }
     }
 
-    fun execute(): Collection<File> {
-        val pool = ForkJoinPool()
-        try {
-            return pool.invoke(this)
-        } finally {
-            pool.shutdown()
-        }
-    }
-
 }

booster-gradle-base/README.md

-# booster-gradle-base
-
-This module provides utilities based on Android gradle plugin.
-
-本模块提供了基于 Android gradle 插件的工具类。
-

booster-gradle-base/build.gradle

-apply from: '../gradle/booster.gradle'
-
-dependencies {
-    compile gradleApi()
-    compile project(':booster-kotlinx')
-    compile 'com.google.auto.service:auto-service:1.0-rc4'
-    compileOnly 'com.android.tools.build:gradle:3.0.0'
-    testCompileOnly 'com.android.tools.build:gradle:3.0.0'
-}

booster-gradle-plugin/build.gradle

@@ -3,7 +3,6 @@ apply from: '../gradle/booster.gradle'
 dependencies {
     kapt 'com.google.auto.service:auto-service:1.0-rc4'
     compile project(':booster-android-gradle-api')
-    compile project(':booster-gradle-base')
     compile project(':booster-task-spi')
     compile project(':booster-transform-spi')
     compile project(':booster-transform-util')

booster-gradle-plugin/src/main/kotlin/com/didiglobal/booster/gradle/BoosterTransformInvocation.kt

@@ -15,6 +15,7 @@ import com.android.build.gradle.internal.publishing.AndroidArtifacts.ArtifactSco
 import com.android.build.gradle.internal.publishing.AndroidArtifacts.ArtifactType.AAR
 import com.android.build.gradle.internal.publishing.AndroidArtifacts.ArtifactType.JAR
 import com.android.build.gradle.internal.publishing.AndroidArtifacts.ConsumedConfigType.RUNTIME_CLASSPATH
+import com.didiglobal.booster.kotlinx.execute
 import com.didiglobal.booster.kotlinx.ifNotEmpty
 import com.didiglobal.booster.transform.ArtifactManager
 import com.didiglobal.booster.transform.Klass
@@ -61,6 +62,8 @@ internal class BoosterTransformInvocation(private val delegate: TransformInvocat
 
     override val klassPool = KlassPoolImpl(runtimeClasspath)
 
+    override val applicationId = delegate.applicationId
+
     override fun hasProperty(name: String): Boolean {
         return project.hasProperty(name)
     }

booster-kotlinx/src/main/kotlin/com/didiglobal/booster/kotlinx/Wildcard.kt

@@ -113,6 +113,8 @@ class Wildcard(private val pattern: String, private val ignoreCase: Boolean = fa
         else -> false
     }
 
+    override fun toString() = this.pattern
+
     private fun checkIndexOf(str: String, strStartIndex: Int, search: String): Int {
         val endIndex = str.length - search.length
         if (endIndex >= strStartIndex) {

booster-kotlinx/src/main/kotlin/com/didiglobal/booster/kotlinx/concurrent.kt

+package com.didiglobal.booster.kotlinx
+
+import java.util.concurrent.ForkJoinPool
+import java.util.concurrent.ForkJoinTask
+
+/**
+ * Execute this task
+ *
+ * @author johnsonlee
+ */
+fun <T> ForkJoinTask<T>.execute(): T {
+    val pool = ForkJoinPool()
+    val result = pool.invoke(this)
+    pool.shutdown()
+    return result
+}

booster-transform-lint/build.gradle

@@ -2,7 +2,7 @@ apply from: '../gradle/booster.gradle'
 
 dependencies {
     kapt "com.google.auto.service:auto-service:1.0-rc4"
-    implementation project(':booster-gradle-base')
+    implementation project(':booster-android-gradle-api')
     implementation project(':booster-transform-asm')
     implementation project(':booster-transform-util')
 }

booster-transform-shared-preferences/build.gradle

@@ -3,7 +3,6 @@ apply from: '../gradle/booster.gradle'
 dependencies {
     kapt "com.google.auto.service:auto-service:1.0-rc4"
     implementation project(':booster-android-gradle-api')
-    implementation project(':booster-gradle-base')
     implementation project(':booster-task-spi')
     implementation project(':booster-transform-asm')
     compileOnly 'com.android.tools.build:gradle:3.0.0'

booster-transform-shrink/README.md

@@ -2,3 +2,23 @@
 
 This module is used for constants shrinking, such as fields in `BuildConfig`, `R$id`, `R$layout`, `R$string`, etc.
 
+## Properties
+
+The following table shows the properties that transformer supports:
+
+| Property                         | Description                                                  | Example                            |
+| -------------------------------- | ------------------------------------------------------------ | ---------------------------------- |
+| `booster.transform.shrink.ignores` | comma separated wildcard patterns to ignore                  | android/\*,androidx/\*             |
+
+The properties can be passthrough the command line as following:
+
+```bash
+./gradlew assembleDebug -Pbooster.transform.shrink.ignores=android/*,androidx/*
+```
+
+or configured in the `gradle.properties`:
+
+```properties
+booster.transform.shrink.ignores=android/*,androidx/*
+```
+

booster-transform-shrink/build.gradle

@@ -3,6 +3,6 @@ apply from: '../gradle/booster.gradle'
 dependencies {
     kapt "com.google.auto.service:auto-service:1.0-rc4"
     implementation project(':booster-aapt2')
-    implementation project(':booster-gradle-base')
+    implementation project(':booster-android-gradle-api')
     implementation project(':booster-transform-asm')
 }

booster-transform-shrink/src/main/kotlin/com/didiglobal/booster/transform/shrink/RCollector.kt

-package com.didiglobal.booster.transform.shrink
-
-import java.io.File
-import java.util.concurrent.RecursiveTask
-
-internal class RCollector(private val root: File) : RecursiveTask<Collection<File>>() {
-
-    override fun compute(): Collection<File> {
-        val tasks = mutableListOf<RecursiveTask<Collection<File>>>()
-        val result = mutableListOf<File>()
-
-        root.listFiles()?.forEach { file ->
-            if (file.isDirectory) {
-                RCollector(file).also { task ->
-                    tasks.add(task)
-                }.fork()
-            } else if ("R.class" == file.name || (file.name.startsWith("R$") && file.name.endsWith(".class"))) {
-                result.add(file)
-            }
-        }
-
-        return result + tasks.flatMap { it.join() }
-    }
-
-}

booster-transform-shrink/src/main/kotlin/com/didiglobal/booster/transform/shrink/ShrinkTransformer.kt

 package com.didiglobal.booster.transform.shrink
 
-import com.didiglobal.booster.kotlinx.BLUE
-import com.didiglobal.booster.kotlinx.RED
-import com.didiglobal.booster.kotlinx.RESET
-import com.didiglobal.booster.kotlinx.YELLOW
+import com.didiglobal.booster.kotlinx.Wildcard
 import com.didiglobal.booster.kotlinx.asIterable
-import com.didiglobal.booster.kotlinx.head
-import com.didiglobal.booster.kotlinx.separatorsToSystem
+import com.didiglobal.booster.kotlinx.execute
+import com.didiglobal.booster.kotlinx.file
+import com.didiglobal.booster.kotlinx.ifNotEmpty
+import com.didiglobal.booster.kotlinx.touch
 import com.didiglobal.booster.transform.ArtifactManager.Companion.JAVAC
 import com.didiglobal.booster.transform.ArtifactManager.Companion.MERGED_RES
 import com.didiglobal.booster.transform.ArtifactManager.Companion.SYMBOL_LIST
-import com.didiglobal.booster.transform.ArtifactManager.Companion.SYMBOL_LIST_WITH_PACKAGE_NAME
 import com.didiglobal.booster.transform.TransformContext
 import com.didiglobal.booster.transform.asm.ClassTransformer
 import com.didiglobal.booster.transform.asm.simpleName
+import com.didiglobal.booster.util.FileFinder
 import com.google.auto.service.AutoService
-import org.gradle.api.logging.Logging
-import org.objectweb.asm.Opcodes
+import org.objectweb.asm.Opcodes.ACC_FINAL
+import org.objectweb.asm.Opcodes.ACC_STATIC
+import org.objectweb.asm.Opcodes.GETSTATIC
 import org.objectweb.asm.tree.ClassNode
 import org.objectweb.asm.tree.FieldInsnNode
 import org.objectweb.asm.tree.FieldNode
 import org.objectweb.asm.tree.LdcInsnNode
-import java.util.concurrent.ForkJoinPool
+import java.io.File
+import java.io.PrintWriter
 
 internal const val R_STYLEABLE = "R\$styleable"
 internal const val R_STYLEABLE_CLASS = "$R_STYLEABLE.class"
 internal const val ANDROID_R = "android/R$"
 internal const val COM_ANDROID_INTERNAL_R = "com/android/internal/R$"
 
-internal val CONST_TYPE_SIGNATURES = setOf("Z", "B", "C", "S", "I", "J", "F", "D", "L/java/lang/String;")
-
-private val logger = Logging.getLogger(ShrinkTransformer::class.java)
-
 /**
  * Represents a class node transformer for constants shrinking
  *
@@ -40,37 +37,75 @@ private val logger = Logging.getLogger(ShrinkTransformer::class.java)
 @AutoService(ClassTransformer::class)
 class ShrinkTransformer : ClassTransformer {
 
-    private lateinit var applicationId: String
-    private lateinit var pkgRStyleable: String
+    private lateinit var appPackage: String
+    private lateinit var appRStyleable: String
+    private lateinit var appRStyleableClass: String
     private lateinit var symbols: SymbolList
     private lateinit var retainedSymbols: Set<String>
+    private lateinit var ignores: Set<Wildcard>
+    private lateinit var logger: PrintWriter
 
     override fun onPreTransform(context: TransformContext) {
-        this.applicationId = context.artifacts.get(SYMBOL_LIST_WITH_PACKAGE_NAME).single().head()!!.replace('.', '/')
+        this.appPackage = context.applicationId.replace('.', '/')
+        this.logger = context.reportsDir.file(Build.ARTIFACT).file(context.name).file("report.txt").touch().printWriter()
         this.symbols = SymbolList.from(context.artifacts.get(SYMBOL_LIST).single())
-        this.pkgRStyleable = "$applicationId/$R_STYLEABLE"
+        this.appRStyleable = "$appPackage/$R_STYLEABLE"
+        this.appRStyleableClass = "$appPackage/$R_STYLEABLE_CLASS"
+        this.ignores = context.getProperty(PROPERTY_IGNORES)?.split(',')?.map { Wildcard(it) }?.toSet() ?: emptySet()
+        val redundant = context.findRedundantR()
+
+        logger.println("$PROPERTY_IGNORES=$ignores\n")
+
+        // Find symbols that should be retained
+        this.retainedSymbols = context.findRetainedSymbols()
+        retainedSymbols.ifNotEmpty { symbols ->
+            logger.println("Retained symbols:")
+            symbols.forEach {
+                logger.println("  - R.id.$it")
+            }
+            logger.println()
+        }
+
+        // Remove redundant R class files
+        redundant.ifNotEmpty { pairs ->
+            val totalSize = redundant.map { it.first.length() }.sum()
+            val maxWidth = redundant.map { it.second.length }.max()?.plus(10) ?: 10
 
-        ForkJoinPool().also { pool ->
-            context.deleteLibraryRs(pool, applicationId)
-            retainedSymbols = context.findRetainedSymbols(pool)
-        }.shutdown()
+            logger.println("Delete files:")
 
-        logger.info("Retained symbols: \n  ${retainedSymbols.joinTo(StringBuilder(), "\n  - ")}")
+            pairs.forEach {
+                if (it.first.delete()) {
+                    logger.println(" - `${it.second}`")
+                }
+            }
+
+            logger.println("-".repeat(maxWidth))
+            logger.println("Total: $totalSize bytes")
+            logger.println()
+        }
     }
 
     override fun transform(context: TransformContext, klass: ClassNode): ClassNode {
         when {
-            klass.name == this.pkgRStyleable -> klass.removeIntFields()
+            this.ignores.any { it.matches(klass.name) } -> {
+                logger.println("Ignore `${klass.name}`")
+                return klass
+            }
+            klass.name == this.appRStyleable -> klass.removeIntFields()
             klass.simpleName == "BuildConfig" -> klass.removeConstantFields()
             else -> replaceSymbolReferenceWithConstant(klass)
         }
         return klass
     }
 
+    override fun onPostTransform(context: TransformContext) {
+        logger.close()
+    }
+
     private fun replaceSymbolReferenceWithConstant(klass: ClassNode) {
         klass.methods.forEach { method ->
             val insns = method.instructions.iterator().asIterable().filter {
-                it.opcode == Opcodes.GETSTATIC
+                it.opcode == GETSTATIC
             }.map {
                 it as FieldInsnNode
             }.filter {
@@ -89,17 +124,66 @@ class ShrinkTransformer : ClassTransformer {
                     method.instructions.insertBefore(field, LdcInsnNode(symbols.getInt(type, field.name)))
                     method.instructions.remove(field)
                 } catch (e: NullPointerException) {
-                    println("$RED ! Unresolvable symbol R.$type.${field.name} : ${klass.name}.${method.name}${method.desc} $RESET")
+                    logger.println("Unresolvable symbol `R.$type.${field.name}` : ${klass.name}.${method.name}${method.desc}")
                 }
             }
 
             // Replace library's R fields with application's R fields
             intArrayFields.forEach { field ->
-                field.owner = "$applicationId/${field.owner.substring(field.owner.lastIndexOf('/') + 1)}"
+                field.owner = "$appPackage/${field.owner.substring(field.owner.lastIndexOf('/') + 1)}"
             }
         }
     }
 
+    private fun TransformContext.findRedundantR(): List<Pair<File, String>> {
+        return artifacts.get(JAVAC).map { classes ->
+            val base = classes.toURI()
+
+            FileFinder(classes) { r ->
+                r.name.startsWith("R") && r.name.endsWith(".class") && (r.name[1] == '$' || r.name.length == 7)
+            }.execute().map { r ->
+                Pair(r, base.relativize(r.toURI()).path)
+            }
+        }.flatten().filter {
+            it.second != appRStyleableClass // keep application's R$styleable.class
+        }.filter { pair ->
+            !ignores.any { it.matches(pair.second) }
+        }
+    }
+
+    private fun ClassNode.removeIntFields() {
+        fields.map {
+            it as FieldNode
+        }.filter { field ->
+            val signature = "$name.${field.name}${field.desc}"
+            !ignores.any { it.matches(signature) }
+        }.filter {
+            it.desc == "I"
+        }.forEach {
+            fields.remove(it)
+            logger.println("Remove `$name.${it.name} : ${it.desc}` = 0x${(it.value as Int).toString(16)}")
+        }
+    }
+
+    private fun ClassNode.removeConstantFields() {
+        fields.map {
+            it as FieldNode
+        }.filter { field ->
+            val signature = "$name.${field.name}${field.desc}"
+            !ignores.any { it.matches(signature) }
+        }.filter {
+            0 != (ACC_STATIC and it.access) && 0 != (ACC_FINAL and it.access) && it.value != null
+        }.forEach {
+            fields.remove(it)
+            logger.println("Remove `$name.${it.name} : ${it.desc}` = ${it.valueAsString()}")
+        }
+    }
+
+}
+
+private fun FieldNode.valueAsString() = when {
+    this.value is String -> "\"${this.value}\""
+    else -> this.value.toString()
 }
 
 /**
@@ -107,51 +191,12 @@ class ShrinkTransformer : ClassTransformer {
  *
  * - attribute `constraint_referenced_ids` in `ConstraintLayout`
  */
-private fun TransformContext.findRetainedSymbols(pool: ForkJoinPool): Set<String> {
+private fun TransformContext.findRetainedSymbols(): Set<String> {
     return artifacts.get(MERGED_RES).map {
-        pool.invoke(RetainedSymbolCollector(it))
+        RetainedSymbolCollector(it).execute()
     }.flatten().toSet()
 }
 
-/**
- * Delete R files of libraries
- */
-private fun TransformContext.deleteLibraryRs(pool: ForkJoinPool, applicationId: String) {
-    return artifacts.get(JAVAC).map {
-        pool.invoke(RCollector(it))
-    }.flatten().filter {
-        // only keep application's R$styleable.class
-        !it.parent.endsWith(applicationId.separatorsToSystem()) || it.name != R_STYLEABLE_CLASS
-    }.forEach {
-        // delete library's R
-        if (it.delete()) {
-            println("$YELLOW x ${it.absolutePath}$RESET")
-        } else {
-            println("$BLUE ? ${it.absolutePath}$RESET")
-        }
-    }
-}
-
-private fun ClassNode.removeIntFields() {
-    fields.map {
-        it as FieldNode
-    }.filter {
-        it.desc == "I"
-    }.forEach {
-        fields.remove(it)
-        println("$YELLOW x $name.${it.name} : ${it.desc}$RESET")
-    }
-}
+private val PROPERTY_PREFIX = Build.ARTIFACT.replace('-', '.')
 
-private fun ClassNode.removeConstantFields() {
-    fields.map {
-        it as FieldNode
-    }.filter {
-        0 != (Opcodes.ACC_STATIC and it.access)
-                && 0 != (Opcodes.ACC_FINAL and it.access)
-                && CONST_TYPE_SIGNATURES.contains(it.desc)
-    }.forEach {
-        fields.remove(it)
-        println("$YELLOW x $name.${it.name} : ${it.desc}$RESET")
-    }
-}
+private val PROPERTY_IGNORES = "$PROPERTY_PREFIX.ignores"

booster-transform-spi/src/main/kotlin/com/didiglobal/booster/transform/TransformContext.kt

@@ -60,6 +60,11 @@ interface TransformContext {
      */
     val klassPool: KlassPool
 
+    /**
+     * The application identifier
+     */
+    val applicationId: String
+
     /**
      * Check if has the specified property. Generally, the property is equivalent to project property
      *

booster-transform-thread/build.gradle

@@ -3,7 +3,6 @@ apply from: '../gradle/booster.gradle'
 dependencies {
     kapt "com.google.auto.service:auto-service:1.0-rc4"
     implementation project(':booster-android-gradle-api')
-    implementation project(':booster-gradle-base')
     implementation project(':booster-task-spi')
     implementation project(':booster-transform-asm')
     compileOnly 'com.android.tools.build:gradle:3.0.0'

booster-transform-toast/build.gradle

@@ -3,7 +3,6 @@ apply from: '../gradle/booster.gradle'
 dependencies {
     kapt "com.google.auto.service:auto-service:1.0-rc4"
     implementation project(':booster-android-gradle-api')
-    implementation project(':booster-gradle-base')
     implementation project(':booster-task-spi')
     implementation project(':booster-transform-asm')
     compileOnly 'com.android.tools.build:gradle:3.0.0'

booster-transform-usage/build.gradle

@@ -2,6 +2,6 @@ apply from: '../gradle/booster.gradle'
 
 dependencies {
     kapt "com.google.auto.service:auto-service:1.0-rc4"
-    compile project(':booster-gradle-base')
+    compile project(':booster-android-gradle-api')
     compile project(':booster-transform-asm')
 }

settings.gradle

@@ -8,7 +8,6 @@ include ':booster-android-gradle-api'
 include ':booster-android-gradle-v3_0'
 include ':booster-android-gradle-v3_2'
 include ':booster-android-gradle-v3_3'
-include ':booster-gradle-base'
 include ':booster-gradle-plugin'
 include ':booster-task-all'
 include ':booster-task-artifact'