fix: handle anonymous class self inlining (#604)

jadx-core/src/main/java/jadx/core/codegen/ClassGen.java

@@ -36,6 +36,7 @@ import jadx.core.utils.CodeGenUtils;
 import jadx.core.utils.ErrorsCounter;
 import jadx.core.utils.Utils;
 import jadx.core.utils.exceptions.CodegenException;
+import jadx.core.utils.exceptions.JadxRuntimeException;
 
 public class ClassGen {
 
@@ -261,6 +262,9 @@ public class ClassGen {
 			try {
 				addMethod(code, mth);
 			} catch (Exception e) {
+				if (mth.getParentClass().getTopParentClass().contains(AFlag.RESTART_CODEGEN)) {
+					throw new JadxRuntimeException("Method generation error", e);
+				}
 				code.newLine().add("/*");
 				code.newLine().addMultiLine(ErrorsCounter.methodError(mth, "Method generation error", e));
 				Utils.appendStackTrace(code, e);

jadx-core/src/main/java/jadx/core/codegen/CodeGen.java

@@ -3,6 +3,7 @@ package jadx.core.codegen;
 import jadx.core.dex.attributes.AFlag;
 import jadx.core.dex.nodes.ClassNode;
 import jadx.core.utils.exceptions.CodegenException;
+import jadx.core.utils.exceptions.JadxRuntimeException;
 
 public class CodeGen {
 
@@ -11,7 +12,18 @@ public class CodeGen {
 			cls.setCode(CodeWriter.EMPTY);
 		} else {
 			ClassGen clsGen = new ClassGen(cls, cls.root().getArgs());
-			cls.setCode(clsGen.makeClass());
+			CodeWriter code;
+			try {
+				code = clsGen.makeClass();
+			} catch (Exception e) {
+				if (cls.contains(AFlag.RESTART_CODEGEN)) {
+					cls.remove(AFlag.RESTART_CODEGEN);
+					code = clsGen.makeClass();
+				} else {
+					throw new JadxRuntimeException("Code generation error", e);
+				}
+			}
+			cls.setCode(code);
 		}
 	}
 

jadx-core/src/main/java/jadx/core/codegen/InsnGen.java

@@ -591,6 +591,14 @@ public class InsnGen {
 	}
 
 	private void inlineAnonymousConstructor(CodeWriter code, ClassNode cls, ConstructorInsn insn) throws CodegenException {
+		if (this.mth.getParentClass() == cls) {
+			cls.remove(AFlag.ANONYMOUS_CLASS);
+			cls.remove(AFlag.DONT_GENERATE);
+			mth.getParentClass().getTopParentClass().add(AFlag.RESTART_CODEGEN);
+			throw new CodegenException("Anonymous inner class unlimited recursion detected."
+					+ " Convert class to inner: " + cls.getClassInfo().getFullName());
+		}
+
 		cls.add(AFlag.DONT_GENERATE);
 		ArgType parent;
 		if (cls.getInterfaces().size() == 1) {

jadx-core/src/main/java/jadx/core/codegen/MethodGen.java

@@ -206,6 +206,9 @@ public class MethodGen {
 				classGen.insertDecompilationProblems(code, mth);
 				addInstructions(code);
 			} catch (Exception e) {
+				if (mth.getParentClass().getTopParentClass().contains(AFlag.RESTART_CODEGEN)) {
+					throw e;
+				}
 				mth.addError("Method code generation error", e);
 				classGen.insertDecompilationProblems(code, mth);
 				addInstructions(code);

jadx-core/src/main/java/jadx/core/dex/attributes/AFlag.java

@@ -15,6 +15,7 @@ public enum AFlag {
 	DONT_WRAP,
 	DONT_INLINE,
 	DONT_GENERATE, // process as usual, but don't output to generated code
+	RESTART_CODEGEN,
 	DONT_RENAME, // do not rename during deobfuscation
 	REMOVE, // can be completely removed
 	ADDED_TO_REGION,

jadx-core/src/main/java/jadx/core/utils/files/InputFile.java

@@ -183,6 +183,9 @@ public class InputFile {
 			if (pathList.isEmpty()) {
 				throw new JadxException("Empty dx output");
 			}
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("result dex files: {}", pathList);
+			}
 			return pathList;
 		} catch (Exception e) {
 			throw new DecodeException("java class to dex conversion error:\n " + e.getMessage(), e);

jadx-core/src/main/java/jadx/core/utils/files/JavaToDex.java

@@ -41,7 +41,7 @@ public class JavaToDex {
 		try (ByteArrayOutputStream out = new ByteArrayOutputStream();
 				ByteArrayOutputStream errOut = new ByteArrayOutputStream()) {
 			DxContext context = new DxContext(out, errOut);
-			Path dir = FileUtils.createTempDir(jar.getFileName().toString());
+			Path dir = FileUtils.createTempDir("jar-to-dex-");
 			DxArgs args = new DxArgs(
 					context,
 					dir.toAbsolutePath().toString(),
@@ -58,7 +58,6 @@ public class JavaToDex {
 					child.toFile().deleteOnExit();
 				}
 			}
-			dir.toFile().deleteOnExit();
 			return list;
 		} catch (Exception e) {
 			throw new JadxException("dx exception: " + e.getMessage(), e);

jadx-core/src/test/java/jadx/tests/api/SmaliTest.java

@@ -49,6 +49,10 @@ public abstract class SmaliTest extends IntegrationTest {
 		return getClassNodeFromFile(outDex, pkg + '.' + clsName);
 	}
 
+	protected ClassNode getClassNodeFromSmaliFiles(String clsName) {
+		return searchCls(loadFromSmaliFiles(), getTestPkg() + '.' + clsName);
+	}
+
 	protected List<ClassNode> loadFromSmaliFiles() {
 		File outDex = createTempFile(".dex");
 		compileSmali(outDex, collectSmaliFiles(getTestPkg(), getTestName()));

jadx-core/src/test/java/jadx/tests/integration/inner/TestIncorrectAnonymousClass.java

+package jadx.tests.integration.inner;
+
+import org.junit.jupiter.api.Test;
+
+import jadx.core.dex.nodes.ClassNode;
+import jadx.tests.api.SmaliTest;
+
+import static jadx.tests.api.utils.JadxMatchers.containsOne;
+import static jadx.tests.api.utils.JadxMatchers.countString;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class TestIncorrectAnonymousClass extends SmaliTest {
+
+	// @formatter:off
+	/*
+		public static class TestCls {
+			public final class 1 {
+				public void invoke() {
+					new 1(); // cause infinite self inline
+				}
+			}
+
+			public void test() {
+				new 1();
+			}
+		}
+	*/
+	// @formatter:on
+
+	@Test
+	public void test() {
+		ClassNode cls = getClassNodeFromSmaliFiles("TestCls");
+		String code = cls.getCode().toString();
+
+		assertThat(code, containsOne("public final class AnonymousClass1 {"));
+		assertThat(code, countString(2, "new AnonymousClass1();"));
+	}
+}

jadx-core/src/test/smali/inner/TestIncorrectAnonymousClass/TestCls$1.smali

+.class public final Linner/TestCls$1;
+.super Ljava/lang/Object;
+
+
+# direct methods
+.method public constructor <init>()V
+    .registers 1
+
+    invoke-direct {p0}, Ljava/lang/Object;-><init>()V
+
+    return-void
+.end method
+
+
+# virtual methods
+.method public invoke()V
+    .registers 2
+
+    new-instance v0, Linner/TestCls$1;
+
+    invoke-direct {v0}, Linner/TestCls$1;-><init>()V
+
+    return-void
+.end method

jadx-core/src/test/smali/inner/TestIncorrectAnonymousClass/TestCls.smali

+.class public Linner/TestCls;
+.super Ljava/lang/Object;
+
+# direct methods
+.method public constructor <init>()V
+    .registers 1
+
+    invoke-direct {p0}, Ljava/lang/Object;-><init>()V
+
+    return-void
+.end method
+
+.method public test()V
+    .registers 2
+
+    new-instance v0, Linner/TestCls$1;
+
+    invoke-direct {v0}, Linner/TestCls$1;-><init>()V
+
+    return-void
+.end method